Merge branch 'worktree-agent-ac3c9345edd29a1c6'
# Conflicts: # app/cli.py # app/jobs/sources.py # app/static/app.css # app/static/companies.js # app/web/api.py # tests/test_web.py
This commit is contained in:
commit
c6246af994
21 files changed
+1017
-155
No files matched your search
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"offset": 2,
|
||||
"limit": 2,
|
||||
"totalFound": 3,
|
||||
"content": [
|
||||
{
|
||||
"id": "744000152001234",
|
||||
"name": "Warehouse Operations Lead",
|
||||
"company": {"identifier": "Acme", "name": "Acme"},
|
||||
"releasedDate": "2026-09-25T07:00:00.000Z",
|
||||
"location": {"city": "Tilburg", "country": "nl", "remote": false, "fullLocation": "Tilburg, Netherlands"},
|
||||
"department": {"id": "3", "label": "Operations"},
|
||||
"function": {"id": "operations", "label": "Operations"}
|
||||
}
|
||||
]
|
||||
}
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"offset": 0,
|
||||
"limit": 100,
|
||||
"totalFound": 2,
|
||||
"limit": 2,
|
||||
"totalFound": 3,
|
||||
"content": [
|
||||
{
|
||||
"id": "744000151326819",
|
||||
|
||||
Vendored
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"total": 3,
|
||||
"results": [
|
||||
{
|
||||
"id": 6125077,
|
||||
"shortcode": "0A1B2C3D4E",
|
||||
"title": "Maintenance Site Team Leader",
|
||||
"remote": false,
|
||||
"location": {
|
||||
"country": "Ireland",
|
||||
"countryCode": "IE",
|
||||
"city": "Dublin",
|
||||
"region": "County Dublin",
|
||||
"display": "Dublin, County Dublin, Ireland"
|
||||
},
|
||||
"state": "published",
|
||||
"published": "2026-09-28T00:00:00.000Z",
|
||||
"type": "full",
|
||||
"department": [
|
||||
"Maintenance"
|
||||
],
|
||||
"workplace": "on_site"
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+2
-1
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"total": 2,
|
||||
"total": 3,
|
||||
"nextPage": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0=",
|
||||
"results": [
|
||||
{
|
||||
"id": 6125026,
|
||||
|
||||
+166
-1
@@ -4,9 +4,12 @@ under `tests/fixtures/ats/`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -160,13 +163,14 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
|
||||
return (FIXTURES / "greenhouse.json").read_bytes()
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
||||
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
||||
|
||||
assert calls == [
|
||||
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
|
||||
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
|
||||
]
|
||||
assert [r["source_id"] for r in rows] == ["1001", "1002"]
|
||||
assert complete # a host that does not page answers with the whole board
|
||||
|
||||
|
||||
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
@@ -177,3 +181,164 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
with pytest.raises(ats._TooLarge):
|
||||
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- paging
|
||||
|
||||
def _page(name: str) -> bytes:
|
||||
return (FIXTURES / name).read_bytes()
|
||||
|
||||
|
||||
def _status(code: int) -> httpx.HTTPStatusError:
|
||||
request = httpx.Request("POST", "https://apply.workable.com/")
|
||||
return httpx.HTTPStatusError(str(code), request=request,
|
||||
response=httpx.Response(code, request=request))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
|
||||
|
||||
def test_workable_follows_next_page_to_the_stated_total(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
calls: list[dict[str, Any] | None] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
calls.append(payload)
|
||||
return _page("workable-2.json" if payload else "workable.json")
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
|
||||
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
|
||||
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
|
||||
assert complete
|
||||
|
||||
|
||||
def test_smartrecruiters_pages_by_offset_to_total_found(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
urls: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
urls.append(url)
|
||||
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
|
||||
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
|
||||
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
|
||||
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
|
||||
assert complete
|
||||
|
||||
|
||||
def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
"""totalFound says 3; the second page is empty, so two were read."""
|
||||
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
return empty if "offset=2" in url else _page("smartrecruiters.json")
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
assert (len(rows), complete) == (2, False)
|
||||
|
||||
|
||||
def test_a_repeated_row_does_not_count_twice_toward_the_total(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
"""A page that shifted under an offset repeats a role and misses one: three
|
||||
rows read, two distinct, against a total of three."""
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
page = json.loads(_page("smartrecruiters.json"))
|
||||
if "offset=2" in url:
|
||||
page["content"] = page["content"][1:]
|
||||
return json.dumps(page).encode()
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
assert (len(rows), complete) == (2, False)
|
||||
|
||||
|
||||
def test_the_page_cap_leaves_a_board_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
calls: list[dict[str, Any] | None] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
calls.append(payload)
|
||||
return _page("workable.json") # always another page
|
||||
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 3)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
assert (len(calls), complete) == (3, False)
|
||||
|
||||
|
||||
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
if payload:
|
||||
raise _status(429)
|
||||
return _page("workable.json")
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
|
||||
|
||||
|
||||
def test_a_429_on_the_first_page_is_a_failure(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
raise _status(429)
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# -------------------------------------------------------- failing boards
|
||||
|
||||
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
||||
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
|
||||
"run": "run-1"})
|
||||
return source
|
||||
|
||||
|
||||
def test_a_failing_board_records_since_and_the_latest_reason(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
source = _source(monkeypatch, _status(404))
|
||||
assert list(source.fetch()) == []
|
||||
first = json.loads(source.state["ats:failing:greenhouse:gone"])
|
||||
assert first["reason"] == "404 Not Found" and first["since"]
|
||||
assert "ats:ok:greenhouse:gone" not in source.state
|
||||
|
||||
# A later failure keeps when the failures began and updates the reason.
|
||||
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
||||
first | {"since": "2026-09-01T00:00:00+00:00"})
|
||||
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
|
||||
later.state = source.state
|
||||
list(later.fetch())
|
||||
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
|
||||
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
|
||||
|
||||
|
||||
def test_an_answer_clears_the_failure_and_stamps_completeness(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
source = _source(monkeypatch, (_rows("greenhouse"), False))
|
||||
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
||||
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
|
||||
assert len(list(source.fetch())) == 2
|
||||
assert "ats:failing:greenhouse:gone" not in source.state
|
||||
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
|
||||
assert source.state["ats:complete:greenhouse:gone"] == ""
|
||||
+217
-17
@@ -4,8 +4,11 @@ touches the network."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
@@ -16,9 +19,10 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext # noqa: E402
|
||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
from app.jobs import cascade, pull # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
|
||||
|
||||
@@ -53,12 +57,14 @@ def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -
|
||||
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
assert sum(len(b["targets"]) for b in batches) == 3
|
||||
assert store.state.get("ats:run", "") != ""
|
||||
run = store.state.get(pull.RUN, "")
|
||||
assert run and {b["run"] for b in batches} == {run}
|
||||
assert pull.running(store) == run
|
||||
|
||||
|
||||
def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None:
|
||||
assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type]
|
||||
assert store.state.get("ats:run", "") == ""
|
||||
assert store.state.get(pull.RUN, "") == ""
|
||||
|
||||
|
||||
def test_run_batch_upserts_every_item_the_plugin_yields(
|
||||
@@ -69,15 +75,15 @@ def test_run_batch_upserts_every_item_the_plugin_yields(
|
||||
def fake_fetch(self: ATSSource) -> object:
|
||||
yield _item("greenhouse", "acme", "1")
|
||||
yield _item("greenhouse", "acme", "2")
|
||||
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
|
||||
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
|
||||
self.state["ats:ids:greenhouse:acme"] = '["1", "2"]'
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
store.state["ats:run"] = "run-1"
|
||||
|
||||
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"])
|
||||
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"], run="run-1")
|
||||
|
||||
assert result == {"roles": 2}
|
||||
assert result == {"roles": 2, "run": "run-1"}
|
||||
assert store.state["ats:ok:greenhouse:acme"] == "run-1"
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"}
|
||||
assert store.roles()[0].subscription_id is not None
|
||||
|
||||
@@ -124,17 +130,17 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
||||
|
||||
# acme answered this run and now lists only "1"; other's board failed
|
||||
# (or its batch never got to it) and stamped nothing.
|
||||
store.state["ats:run"] = "run-1"
|
||||
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
||||
store.state["ats:complete:greenhouse:acme"] = "1"
|
||||
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
||||
|
||||
class FakeTally:
|
||||
results = ({"roles": 1},)
|
||||
results = ({"roles": 1, "run": "run-1"},)
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1,
|
||||
"closed": 1}
|
||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
||||
"roles": 1, "closed": 1}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
||||
|
||||
|
||||
@@ -146,15 +152,209 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
||||
|
||||
# A stale stamp from a run that is not this one — e.g. this run's batch
|
||||
# for this board never finished.
|
||||
store.state["ats:run"] = "run-2"
|
||||
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
||||
store.state["ats:ids:greenhouse:acme"] = "[]"
|
||||
|
||||
class FakeTally:
|
||||
results: tuple[dict[str, int], ...] = ()
|
||||
results = ({"roles": 0, "run": "run-2"},)
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0,
|
||||
"closed": 0}
|
||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
||||
"roles": 0, "closed": 0}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||
|
||||
|
||||
# ---------------------------------------------- whole pulls, fetch stubbed
|
||||
|
||||
def _row(job_id: str) -> dict[str, object]:
|
||||
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
|
||||
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
|
||||
"description": "", "remote": None, "salary": ""}
|
||||
|
||||
|
||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
|
||||
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, object]:
|
||||
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _open(store: Store) -> set[str]:
|
||||
return {r.source_id for r in store.roles()}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def acme(store: Store) -> Store:
|
||||
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
||||
board="workable")
|
||||
return store
|
||||
|
||||
|
||||
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
||||
_pull(acme)
|
||||
# A 429 on page two: page one's role is stored again, the other stays open.
|
||||
_answer(monkeypatch, ([_row("1"), _row("3")], False))
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
||||
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
|
||||
|
||||
|
||||
def test_a_failing_board_keeps_its_roles_before_stale_days(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(acme)
|
||||
_answer(monkeypatch, ValueError("gone"))
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
assert (result["failed"], result["closed"]) == (1, 0)
|
||||
since, reason = pull.failing_boards(acme)["workable:acme"]
|
||||
assert since and reason == "ValueError: gone"
|
||||
|
||||
|
||||
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
||||
_pull(acme)
|
||||
_answer(monkeypatch, ValueError("404 for good"))
|
||||
_pull(acme)
|
||||
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
||||
timespec="seconds")
|
||||
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
||||
{"since": old, "reason": "404 Not Found"})
|
||||
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == set()
|
||||
assert result["closed"] == 2
|
||||
# Still failing, and still the date the failures began.
|
||||
assert pull.failing_boards(acme)["workable:acme"][0] == old
|
||||
|
||||
_answer(monkeypatch, ([_row("2")], True))
|
||||
_pull(acme)
|
||||
assert _open(acme) == {"workable:acme:2"}
|
||||
assert pull.failing_boards(acme) == {}
|
||||
|
||||
|
||||
# ------------------------------------------------------ one pull at a time
|
||||
|
||||
def _ago(**delta: float) -> str:
|
||||
return (datetime.now(UTC) - timedelta(**delta)).isoformat(timespec="microseconds")
|
||||
|
||||
|
||||
def test_a_pull_during_a_running_one_is_refused_and_queues_nothing(acme: Store) -> None:
|
||||
first = pull.claim_run(acme)
|
||||
assert first and pull.running(acme) == first
|
||||
|
||||
asked: list[object] = []
|
||||
said: list[object] = []
|
||||
context = RunContext(on_task=lambda task: asked.append(task) or True,
|
||||
on_notice=said.append)
|
||||
result = cascade.run_pull(FakeConfig(), acme, context=context)
|
||||
|
||||
assert result == {"skipped": pull.ALREADY_RUNNING, "running": first}
|
||||
assert asked == []
|
||||
assert [n.code for n in said] == ["pull.running"] # type: ignore[attr-defined]
|
||||
# A chain started without the job's claim (a test, a script) finds
|
||||
# nothing to queue either, and the running run's marker stands.
|
||||
assert pull._boards(FakeConfig(), acme) == [] # type: ignore[arg-type]
|
||||
assert acme.state[pull.RUN] == first
|
||||
|
||||
|
||||
def test_a_pull_starts_once_the_last_one_settled(acme: Store,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(acme)
|
||||
assert pull.running(acme) == ""
|
||||
|
||||
asked: list[object] = []
|
||||
context = RunContext(on_task=lambda task: asked.append(task) or True)
|
||||
result = cascade.run_pull(FakeConfig(), acme, context=context)
|
||||
assert result["run"] == acme.state[pull.RUN] and len(asked) == 1
|
||||
# The chain's finder reads under the job's claim instead of claiming again.
|
||||
[batch] = asked[0].finder(FakeConfig(), acme) # type: ignore[attr-defined]
|
||||
assert batch["run"] == result["run"]
|
||||
|
||||
|
||||
def test_starters_claiming_at_once_give_one_run(tmp_path: Path) -> None:
|
||||
"""Eight starters, each on its own connection to one file, as two
|
||||
processes would be, released together: one claim wins."""
|
||||
Store(tmp_path / "shared.db")
|
||||
stores = [Store(tmp_path / "shared.db") for _ in range(8)]
|
||||
barrier = threading.Barrier(len(stores))
|
||||
|
||||
def claim(store: Store) -> str:
|
||||
barrier.wait()
|
||||
return pull.claim_run(store)
|
||||
|
||||
with ThreadPoolExecutor(len(stores)) as pool:
|
||||
claims = list(pool.map(claim, stores))
|
||||
assert sum(bool(c) for c in claims) == 1
|
||||
|
||||
|
||||
def test_an_abandoned_run_is_taken_over_after_the_threshold(acme: Store) -> None:
|
||||
hours = pull.ABANDONED.total_seconds() / 3600
|
||||
acme.state[pull.RUN] = _ago(hours=hours / 2)
|
||||
assert pull.claim_run(acme) == ""
|
||||
acme.state[pull.RUN] = _ago(hours=hours + 0.1)
|
||||
taken = pull.claim_run(acme)
|
||||
assert taken and acme.state[pull.RUN] == taken
|
||||
|
||||
|
||||
def test_a_batch_stamps_with_the_marker_it_was_handed(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
[batch] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
||||
acme.state[pull.RUN] = "a later run"
|
||||
pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
|
||||
assert acme.state["ats:ok:workable:acme"] == batch["run"]
|
||||
|
||||
|
||||
def test_run_a_settle_does_not_close_on_run_b_ids(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Run A read the board in full; run B, which took over, read it again and
|
||||
listed less before A's settle ran. A's settle compares with A's marker, so
|
||||
B's stamp and B's ids close nothing, and A's settle leaves B's claim held;
|
||||
B's own settle is the one that closes."""
|
||||
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
||||
[a] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
||||
a_result = pull.run_batch(FakeConfig(), acme, **a) # type: ignore[arg-type]
|
||||
|
||||
acme.state[pull.RUN] = _ago(hours=2) # A looks abandoned
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
[b] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
||||
assert b["run"] != a["run"]
|
||||
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
||||
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
||||
assert (result["failed"], result["closed"]) == (1, 0)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||
assert pull.running(acme) == b["run"]
|
||||
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(results=(b_result,))) # type: ignore[arg-type]
|
||||
assert result["closed"] == 1
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
assert pull.running(acme) == ""
|
||||
|
||||
|
||||
def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> None:
|
||||
"""A batch that raised has no result, but its payload still names the run."""
|
||||
run = pull.claim_run(acme)
|
||||
acme.state["ats:ok:workable:acme"] = run
|
||||
acme.state["ats:complete:workable:acme"] = "1"
|
||||
acme.state["ats:ids:workable:acme"] = "[]"
|
||||
failed = type("Job", (), {"payload": {"targets": ["workable:acme"], "run": run},
|
||||
"result": None, "state": "failed"})()
|
||||
context = RunContext(waited_for=(failed,))
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
||||
assert result["boards"] == 1 and pull.running(acme) == ""
|
||||
+23
-1
@@ -633,7 +633,7 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
||||
published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
|
||||
yield item
|
||||
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
|
||||
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
|
||||
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
@@ -658,6 +658,28 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
||||
assert work == ["pull", "pull_batch", "embed", "score", "score"]
|
||||
assert all(j.state == "done" for j in jobs)
|
||||
assert len(store.scores(agent.id)) == 1
|
||||
# The board was stamped with the run the `pull` job claimed, and the
|
||||
# settle let that run go, so the next pull may start.
|
||||
run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr]
|
||||
assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"]
|
||||
|
||||
|
||||
def test_the_pull_job_queues_nothing_while_another_pull_runs(
|
||||
store: Store, config: Config, tmp_path: Path) -> None:
|
||||
from app.jobs import pull
|
||||
|
||||
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||
board="greenhouse")
|
||||
running = pull.claim_run(store)
|
||||
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
|
||||
workers={"cpu": 1, "http": 1, "llm": 1})
|
||||
started = queue.enqueue("pull")
|
||||
queue.drain(timeout=60)
|
||||
|
||||
done = queue.jobs.get(started.id) # type: ignore[union-attr]
|
||||
assert done is not None and done.state == "done"
|
||||
assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
|
||||
assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"]
|
||||
|
||||
|
||||
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None:
|
||||
|
||||
+62
-16
@@ -4,6 +4,7 @@ searches. No network: Jobindex's feed is a trimmed copy of a real one
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -20,6 +21,7 @@ from plugins import search # noqa: E402
|
||||
from plugins.search import SearchSource # noqa: E402
|
||||
from app.config import SearchSeed # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
from pipeline import Tally # noqa: E402
|
||||
from app.models import country_of, remote_of # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
|
||||
@@ -139,16 +141,68 @@ def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> N
|
||||
return b"short" if params["page"] == 3 else b"full"
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setattr(search, "parse_jobindex",
|
||||
lambda body: full[:5] if body == b"short" else full)
|
||||
monkeypatch.setattr(search, "_jobindex_page",
|
||||
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
|
||||
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
|
||||
|
||||
# Stopped by the page limit with every page full: not complete.
|
||||
pages.clear()
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert (pages, complete) == ([1, 2], False)
|
||||
|
||||
|
||||
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A full page whose parser dropped one item still has 20 items, so the
|
||||
search reads on instead of calling itself complete at 19 rows."""
|
||||
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
||||
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
|
||||
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
|
||||
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert not complete
|
||||
|
||||
|
||||
def test_a_jobindex_answer_without_a_channel_raises() -> None:
|
||||
with pytest.raises(ValueError, match="channel"):
|
||||
search.parse_jobindex(b"<rss version='2.0'></rss>")
|
||||
|
||||
|
||||
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
|
||||
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
|
||||
asked: list[int] = []
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
page = int(url.rsplit("/", 1)[1])
|
||||
asked.append(page)
|
||||
start = (page - 1) * search.ADZUNA_PAGE
|
||||
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
|
||||
body: dict[str, Any] = {"results": results}
|
||||
if count is not None:
|
||||
body["count"] = count
|
||||
return json.dumps(body).encode()
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setenv("ADZUNA_APP_ID", "id")
|
||||
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
|
||||
return asked
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
|
||||
def test_an_adzuna_search_is_complete_when_it_read_its_count(
|
||||
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
|
||||
_adzuna(monkeypatch, count)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
assert (len(rows), done) == (60, complete)
|
||||
|
||||
|
||||
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
|
||||
|
||||
|
||||
def test_provider_of() -> None:
|
||||
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
|
||||
assert search.provider_of("adzuna:berlin") == "adzuna"
|
||||
@@ -176,14 +230,9 @@ def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, Any]:
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
for batch in batches:
|
||||
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
|
||||
class Tally:
|
||||
results: ClassVar[list[dict[str, Any]]] = []
|
||||
|
||||
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
|
||||
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_a_complete_search_closes_what_it_stopped_listing(
|
||||
@@ -253,13 +302,10 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
||||
board="jobindex", params={"q": "backend"})
|
||||
_stub_search(monkeypatch, ["h1"], complete=False)
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
||||
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
||||
store.update_subscription(sub.id, subscribed=False, fetch=False)
|
||||
|
||||
class Tally:
|
||||
results: ClassVar[list[dict[str, Any]]] = []
|
||||
|
||||
pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
|
||||
pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
||||
assert store.roles() == []
|
||||
# And a batch that starts after the switch reads nothing.
|
||||
assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type]
|
||||
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
||||
@@ -36,11 +36,11 @@ class Boards:
|
||||
self.listed = {k.replace("_", ":", 1): v for k, v in listed.items()}
|
||||
self.fetched: list[str] = []
|
||||
|
||||
def __call__(self, target: str) -> list[dict[str, Any]]:
|
||||
def __call__(self, target: str) -> tuple[list[dict[str, Any]], bool]:
|
||||
self.fetched.append(target)
|
||||
if target not in self.listed:
|
||||
raise OSError("connection refused")
|
||||
return [{"source_id": str(i)} for i in range(self.listed[target])]
|
||||
return [{"source_id": str(i)} for i in range(self.listed[target])], True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
+30
-3
@@ -4,6 +4,7 @@ worker beyond starting and stopping it with the app."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -396,7 +397,7 @@ def test_companies_lists_board_and_open_role_count(
|
||||
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
|
||||
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
|
||||
"ats": "greenhouse", "ats_slug": "", "careers_url": "",
|
||||
"country": "DK"}
|
||||
"country": "DK", "failing_since": "", "failing_reason": ""}
|
||||
[other] = [c for c in res.json() if c["key"] == "other"]
|
||||
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
|
||||
"", None, None, "")
|
||||
@@ -405,6 +406,32 @@ def test_companies_lists_board_and_open_role_count(
|
||||
"", None, "successfactors", "https://novo.example/careers")
|
||||
|
||||
|
||||
def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""What the pull recorded, on `/api/companies` and beside `app companies
|
||||
check`'s live result."""
|
||||
import plugins.ats as ats
|
||||
from app.cli import check_companies
|
||||
|
||||
store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
||||
store.add_company("Acme", key="acme", ats="greenhouse")
|
||||
store.state["ats:failing:greenhouse:acme"] = json.dumps(
|
||||
{"since": "2026-09-01T06:00:00+00:00", "reason": "404 Not Found"})
|
||||
|
||||
[row] = client.get("/api/companies").json()
|
||||
assert (row["failing_since"], row["failing_reason"]) == (
|
||||
"2026-09-01T06:00:00+00:00", "404 Not Found")
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
|
||||
assert check_companies(store) == 0
|
||||
assert capsys.readouterr().out.splitlines() == [
|
||||
"greenhouse:acme (greenhouse:acme): now 1 roles; "
|
||||
"pull: failing since 2026-09-01: 404 Not Found",
|
||||
"checked 1 boards, 0 dead, 1 failing in the pull",
|
||||
]
|
||||
|
||||
|
||||
def test_set_affinity_validates_updates_and_enqueues_a_score(
|
||||
client: TestClient, store: Store, queue: Any) -> None:
|
||||
store.add_company("Acme", key="acme")
|
||||
@@ -913,7 +940,7 @@ def test_a_board_is_followed_only_if_it_answers(
|
||||
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import plugins.ats as ats
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: [{"source_id": "1"}])
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
|
||||
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
|
||||
assert res.status_code == 201
|
||||
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
|
||||
@@ -940,7 +967,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
|
||||
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import plugins.ats as ats
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: []) # between openings
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) # between openings
|
||||
store.add_company("Acme", key="acme", ats="teamtailor", ats_slug="acme")
|
||||
|
||||
res = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"})
|
||||
|
||||
Reference in new issue
Block a user