Merge branch 'worktree-agent-ac3c9345edd29a1c6'

# Conflicts:
#	app/cli.py
#	app/jobs/sources.py
#	app/static/app.css
#	app/static/companies.js
#	app/web/api.py
#	tests/test_web.py
This commit is contained in:
Wolf committed 2026-09-29 14:15:10 +02:00
commit c6246af994
21 files changed
+1017 -155

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
{
"offset": 2,
"limit": 2,
"totalFound": 3,
"content": [
{
"id": "744000152001234",
"name": "Warehouse Operations Lead",
"company": {"identifier": "Acme", "name": "Acme"},
"releasedDate": "2026-09-25T07:00:00.000Z",
"location": {"city": "Tilburg", "country": "nl", "remote": false, "fullLocation": "Tilburg, Netherlands"},
"department": {"id": "3", "label": "Operations"},
"function": {"id": "operations", "label": "Operations"}
}
]
}
+2 -2
View File
@@ -1,7 +1,7 @@
{
"offset": 0,
"limit": 100,
"totalFound": 2,
"limit": 2,
"totalFound": 3,
"content": [
{
"id": "744000151326819",
+25
View File
@@ -0,0 +1,25 @@
{
"total": 3,
"results": [
{
"id": 6125077,
"shortcode": "0A1B2C3D4E",
"title": "Maintenance Site Team Leader",
"remote": false,
"location": {
"country": "Ireland",
"countryCode": "IE",
"city": "Dublin",
"region": "County Dublin",
"display": "Dublin, County Dublin, Ireland"
},
"state": "published",
"published": "2026-09-28T00:00:00.000Z",
"type": "full",
"department": [
"Maintenance"
],
"workplace": "on_site"
}
]
}
+2 -1
View File
@@ -1,5 +1,6 @@
{
"total": 2,
"total": 3,
"nextPage": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0=",
"results": [
{
"id": 6125026,
+166 -1
View File
@@ -4,9 +4,12 @@ under `tests/fixtures/ats/`."""
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
@@ -160,13 +163,14 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
return (FIXTURES / "greenhouse.json").read_bytes()
monkeypatch.setattr(ats, "_get", fake_get)
rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
assert calls == [
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
]
assert [r["source_id"] for r in rows] == ["1001", "1002"]
assert complete # a host that does not page answers with the whole board
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
@@ -177,3 +181,164 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(ats._TooLarge):
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
# ----------------------------------------------------------------- paging
def _page(name: str) -> bytes:
return (FIXTURES / name).read_bytes()
def _status(code: int) -> httpx.HTTPStatusError:
request = httpx.Request("POST", "https://apply.workable.com/")
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
@pytest.fixture
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def test_workable_follows_next_page_to_the_stated_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable-2.json" if payload else "workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
assert complete
def test_smartrecruiters_pages_by_offset_to_total_found(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
urls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
urls.append(url)
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
assert complete
def test_a_board_that_ends_short_of_its_total_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""totalFound says 3; the second page is empty, so two were read."""
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
def fake_get(client, url, host, slug, *, method="GET", payload=None):
return empty if "offset=2" in url else _page("smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_a_repeated_row_does_not_count_twice_toward_the_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""A page that shifted under an offset repeats a role and misses one: three
rows read, two distinct, against a total of three."""
def fake_get(client, url, host, slug, *, method="GET", payload=None):
page = json.loads(_page("smartrecruiters.json"))
if "offset=2" in url:
page["content"] = page["content"][1:]
return json.dumps(page).encode()
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_the_page_cap_leaves_a_board_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable.json") # always another page
monkeypatch.setattr(ats, "MAX_PAGES", 3)
monkeypatch.setattr(ats, "_get", fake_get)
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert (len(calls), complete) == (3, False)
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
if payload:
raise _status(429)
return _page("workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
def test_a_429_on_the_first_page_is_a_failure(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
raise _status(429)
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(httpx.HTTPStatusError):
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
# -------------------------------------------------------- failing boards
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
def fake(client: object, target: str) -> object:
if isinstance(answer, Exception):
raise answer
return answer
monkeypatch.setattr(ats, "_fetch_rows", fake)
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
"run": "run-1"})
return source
def test_a_failing_board_records_since_and_the_latest_reason(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, _status(404))
assert list(source.fetch()) == []
first = json.loads(source.state["ats:failing:greenhouse:gone"])
assert first["reason"] == "404 Not Found" and first["since"]
assert "ats:ok:greenhouse:gone" not in source.state
# A later failure keeps when the failures began and updates the reason.
source.state["ats:failing:greenhouse:gone"] = json.dumps(
first | {"since": "2026-09-01T00:00:00+00:00"})
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
later.state = source.state
list(later.fetch())
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
def test_an_answer_clears_the_failure_and_stamps_completeness(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, (_rows("greenhouse"), False))
source.state["ats:failing:greenhouse:gone"] = json.dumps(
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
assert len(list(source.fetch())) == 2
assert "ats:failing:greenhouse:gone" not in source.state
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
assert source.state["ats:complete:greenhouse:gone"] == ""
+217 -17
View File
@@ -4,8 +4,11 @@ touches the network."""
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import ClassVar
@@ -16,9 +19,10 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext # noqa: E402
from pipeline import Cancelled, RunContext, Tally # noqa: E402
from plugins import ats # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.jobs import pull # noqa: E402
from app.jobs import cascade, pull # noqa: E402
from app.storage import Store # noqa: E402
@@ -53,12 +57,14 @@ def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
assert sum(len(b["targets"]) for b in batches) == 3
assert store.state.get("ats:run", "") != ""
run = store.state.get(pull.RUN, "")
assert run and {b["run"] for b in batches} == {run}
assert pull.running(store) == run
def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None:
assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type]
assert store.state.get("ats:run", "") == ""
assert store.state.get(pull.RUN, "") == ""
def test_run_batch_upserts_every_item_the_plugin_yields(
@@ -69,15 +75,15 @@ def test_run_batch_upserts_every_item_the_plugin_yields(
def fake_fetch(self: ATSSource) -> object:
yield _item("greenhouse", "acme", "1")
yield _item("greenhouse", "acme", "2")
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
self.state["ats:ids:greenhouse:acme"] = '["1", "2"]'
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
store.state["ats:run"] = "run-1"
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"])
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"], run="run-1")
assert result == {"roles": 2}
assert result == {"roles": 2, "run": "run-1"}
assert store.state["ats:ok:greenhouse:acme"] == "run-1"
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"}
assert store.roles()[0].subscription_id is not None
@@ -124,17 +130,17 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
# acme answered this run and now lists only "1"; other's board failed
# (or its batch never got to it) and stamped nothing.
store.state["ats:run"] = "run-1"
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:complete:greenhouse:acme"] = "1"
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
class FakeTally:
results = ({"roles": 1},)
results = ({"roles": 1, "run": "run-1"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1,
"closed": 1}
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 1, "closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -146,15 +152,209 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
# A stale stamp from a run that is not this one — e.g. this run's batch
# for this board never finished.
store.state["ats:run"] = "run-2"
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:ids:greenhouse:acme"] = "[]"
class FakeTally:
results: tuple[dict[str, int], ...] = ()
results = ({"roles": 0, "run": "run-2"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0,
"closed": 0}
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 0, "closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
# ---------------------------------------------- whole pulls, fetch stubbed
def _row(job_id: str) -> dict[str, object]:
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
"description": "", "remote": None, "salary": ""}
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
def fake(client: object, target: str) -> object:
if isinstance(answer, Exception):
raise answer
return answer
monkeypatch.setattr(ats, "_fetch_rows", fake)
def _pull(store: Store) -> dict[str, object]:
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
def _open(store: Store) -> set[str]:
return {r.source_id for r in store.roles()}
@pytest.fixture
def acme(store: Store) -> Store:
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
board="workable")
return store
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
# A 429 on page two: page one's role is stored again, the other stays open.
_answer(monkeypatch, ([_row("1"), _row("3")], False))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
def test_a_failing_board_keeps_its_roles_before_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
_answer(monkeypatch, ValueError("gone"))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1"}
assert (result["failed"], result["closed"]) == (1, 0)
since, reason = pull.failing_boards(acme)["workable:acme"]
assert since and reason == "ValueError: gone"
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
_answer(monkeypatch, ValueError("404 for good"))
_pull(acme)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
timespec="seconds")
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
{"since": old, "reason": "404 Not Found"})
result = _pull(acme)
assert _open(acme) == set()
assert result["closed"] == 2
# Still failing, and still the date the failures began.
assert pull.failing_boards(acme)["workable:acme"][0] == old
_answer(monkeypatch, ([_row("2")], True))
_pull(acme)
assert _open(acme) == {"workable:acme:2"}
assert pull.failing_boards(acme) == {}
# ------------------------------------------------------ one pull at a time
def _ago(**delta: float) -> str:
return (datetime.now(UTC) - timedelta(**delta)).isoformat(timespec="microseconds")
def test_a_pull_during_a_running_one_is_refused_and_queues_nothing(acme: Store) -> None:
first = pull.claim_run(acme)
assert first and pull.running(acme) == first
asked: list[object] = []
said: list[object] = []
context = RunContext(on_task=lambda task: asked.append(task) or True,
on_notice=said.append)
result = cascade.run_pull(FakeConfig(), acme, context=context)
assert result == {"skipped": pull.ALREADY_RUNNING, "running": first}
assert asked == []
assert [n.code for n in said] == ["pull.running"] # type: ignore[attr-defined]
# A chain started without the job's claim (a test, a script) finds
# nothing to queue either, and the running run's marker stands.
assert pull._boards(FakeConfig(), acme) == [] # type: ignore[arg-type]
assert acme.state[pull.RUN] == first
def test_a_pull_starts_once_the_last_one_settled(acme: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
assert pull.running(acme) == ""
asked: list[object] = []
context = RunContext(on_task=lambda task: asked.append(task) or True)
result = cascade.run_pull(FakeConfig(), acme, context=context)
assert result["run"] == acme.state[pull.RUN] and len(asked) == 1
# The chain's finder reads under the job's claim instead of claiming again.
[batch] = asked[0].finder(FakeConfig(), acme) # type: ignore[attr-defined]
assert batch["run"] == result["run"]
def test_starters_claiming_at_once_give_one_run(tmp_path: Path) -> None:
"""Eight starters, each on its own connection to one file, as two
processes would be, released together: one claim wins."""
Store(tmp_path / "shared.db")
stores = [Store(tmp_path / "shared.db") for _ in range(8)]
barrier = threading.Barrier(len(stores))
def claim(store: Store) -> str:
barrier.wait()
return pull.claim_run(store)
with ThreadPoolExecutor(len(stores)) as pool:
claims = list(pool.map(claim, stores))
assert sum(bool(c) for c in claims) == 1
def test_an_abandoned_run_is_taken_over_after_the_threshold(acme: Store) -> None:
hours = pull.ABANDONED.total_seconds() / 3600
acme.state[pull.RUN] = _ago(hours=hours / 2)
assert pull.claim_run(acme) == ""
acme.state[pull.RUN] = _ago(hours=hours + 0.1)
taken = pull.claim_run(acme)
assert taken and acme.state[pull.RUN] == taken
def test_a_batch_stamps_with_the_marker_it_was_handed(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
[batch] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
acme.state[pull.RUN] = "a later run"
pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
assert acme.state["ats:ok:workable:acme"] == batch["run"]
def test_run_a_settle_does_not_close_on_run_b_ids(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""Run A read the board in full; run B, which took over, read it again and
listed less before A's settle ran. A's settle compares with A's marker, so
B's stamp and B's ids close nothing, and A's settle leaves B's claim held;
B's own settle is the one that closes."""
_answer(monkeypatch, ([_row("1"), _row("2")], True))
[a] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
a_result = pull.run_batch(FakeConfig(), acme, **a) # type: ignore[arg-type]
acme.state[pull.RUN] = _ago(hours=2) # A looks abandoned
_answer(monkeypatch, ([_row("1")], True))
[b] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
assert b["run"] != a["run"]
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
assert (result["failed"], result["closed"]) == (1, 0)
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
assert pull.running(acme) == b["run"]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(b_result,))) # type: ignore[arg-type]
assert result["closed"] == 1
assert _open(acme) == {"workable:acme:1"}
assert pull.running(acme) == ""
def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> None:
"""A batch that raised has no result, but its payload still names the run."""
run = pull.claim_run(acme)
acme.state["ats:ok:workable:acme"] = run
acme.state["ats:complete:workable:acme"] = "1"
acme.state["ats:ids:workable:acme"] = "[]"
failed = type("Job", (), {"payload": {"targets": ["workable:acme"], "run": run},
"result": None, "state": "failed"})()
context = RunContext(waited_for=(failed,))
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
assert result["boards"] == 1 and pull.running(acme) == ""
+23 -1
View File
@@ -633,7 +633,7 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
yield item
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
@@ -658,6 +658,28 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
assert work == ["pull", "pull_batch", "embed", "score", "score"]
assert all(j.state == "done" for j in jobs)
assert len(store.scores(agent.id)) == 1
# The board was stamped with the run the `pull` job claimed, and the
# settle let that run go, so the next pull may start.
run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr]
assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"]
def test_the_pull_job_queues_nothing_while_another_pull_runs(
store: Store, config: Config, tmp_path: Path) -> None:
from app.jobs import pull
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
running = pull.claim_run(store)
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
workers={"cpu": 1, "http": 1, "llm": 1})
started = queue.enqueue("pull")
queue.drain(timeout=60)
done = queue.jobs.get(started.id) # type: ignore[union-attr]
assert done is not None and done.state == "done"
assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"]
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None:
+62 -16
View File
@@ -4,6 +4,7 @@ searches. No network: Jobindex's feed is a trimmed copy of a real one
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -20,6 +21,7 @@ from plugins import search # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402
@@ -139,16 +141,68 @@ def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> N
return b"short" if params["page"] == 3 else b"full"
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setattr(search, "parse_jobindex",
lambda body: full[:5] if body == b"short" else full)
monkeypatch.setattr(search, "_jobindex_page",
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
# Stopped by the page limit with every page full: not complete.
pages.clear()
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert (pages, complete) == ([1, 2], False)
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
monkeypatch: pytest.MonkeyPatch) -> None:
"""A full page whose parser dropped one item still has 20 items, so the
search reads on instead of calling itself complete at 19 rows."""
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert not complete
def test_a_jobindex_answer_without_a_channel_raises() -> None:
with pytest.raises(ValueError, match="channel"):
search.parse_jobindex(b"<rss version='2.0'></rss>")
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
asked: list[int] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
page = int(url.rsplit("/", 1)[1])
asked.append(page)
start = (page - 1) * search.ADZUNA_PAGE
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
body: dict[str, Any] = {"results": results}
if count is not None:
body["count"] = count
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setenv("ADZUNA_APP_ID", "id")
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
return asked
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
def test_an_adzuna_search_is_complete_when_it_read_its_count(
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
_adzuna(monkeypatch, count)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (60, complete)
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
def test_provider_of() -> None:
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
assert search.provider_of("adzuna:berlin") == "adzuna"
@@ -176,14 +230,9 @@ def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool
def _pull(store: Store) -> dict[str, Any]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
for batch in batches:
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
def test_a_complete_search_closes_what_it_stopped_listing(
@@ -253,13 +302,10 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
board="jobindex", params={"q": "backend"})
_stub_search(monkeypatch, ["h1"], complete=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
store.update_subscription(sub.id, subscribed=False, fetch=False)
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
assert store.roles() == []
# And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type]
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
+2 -2
View File
@@ -36,11 +36,11 @@ class Boards:
self.listed = {k.replace("_", ":", 1): v for k, v in listed.items()}
self.fetched: list[str] = []
def __call__(self, target: str) -> list[dict[str, Any]]:
def __call__(self, target: str) -> tuple[list[dict[str, Any]], bool]:
self.fetched.append(target)
if target not in self.listed:
raise OSError("connection refused")
return [{"source_id": str(i)} for i in range(self.listed[target])]
return [{"source_id": str(i)} for i in range(self.listed[target])], True
@pytest.fixture
+30 -3
View File
@@ -4,6 +4,7 @@ worker beyond starting and stopping it with the app."""
from __future__ import annotations
import json
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -396,7 +397,7 @@ def test_companies_lists_board_and_open_role_count(
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
"ats": "greenhouse", "ats_slug": "", "careers_url": "",
"country": "DK"}
"country": "DK", "failing_since": "", "failing_reason": ""}
[other] = [c for c in res.json() if c["key"] == "other"]
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
"", None, None, "")
@@ -405,6 +406,32 @@ def test_companies_lists_board_and_open_role_count(
"", None, "successfactors", "https://novo.example/careers")
def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,
capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch) -> None:
"""What the pull recorded, on `/api/companies` and beside `app companies
check`'s live result."""
import plugins.ats as ats
from app.cli import check_companies
store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme", ats="greenhouse")
store.state["ats:failing:greenhouse:acme"] = json.dumps(
{"since": "2026-09-01T06:00:00+00:00", "reason": "404 Not Found"})
[row] = client.get("/api/companies").json()
assert (row["failing_since"], row["failing_reason"]) == (
"2026-09-01T06:00:00+00:00", "404 Not Found")
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
assert check_companies(store) == 0
assert capsys.readouterr().out.splitlines() == [
"greenhouse:acme (greenhouse:acme): now 1 roles; "
"pull: failing since 2026-09-01: 404 Not Found",
"checked 1 boards, 0 dead, 1 failing in the pull",
]
def test_set_affinity_validates_updates_and_enqueues_a_score(
client: TestClient, store: Store, queue: Any) -> None:
store.add_company("Acme", key="acme")
@@ -913,7 +940,7 @@ def test_a_board_is_followed_only_if_it_answers(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: [{"source_id": "1"}])
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
assert res.status_code == 201
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
@@ -940,7 +967,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: []) # between openings
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) # between openings
store.add_company("Acme", key="acme", ats="teamtailor", ats_slug="acme")
res = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"})