Paged boards are read to their stated total, and a board failing for 21 days closes

Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:02:49 +02:00
1 parent 0558ef19ed
commit c7da12b93d
18 files changed
+695 -91

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
{
"offset": 2,
"limit": 2,
"totalFound": 3,
"content": [
{
"id": "744000152001234",
"name": "Warehouse Operations Lead",
"company": {"identifier": "Acme", "name": "Acme"},
"releasedDate": "2026-09-25T07:00:00.000Z",
"location": {"city": "Tilburg", "country": "nl", "remote": false, "fullLocation": "Tilburg, Netherlands"},
"department": {"id": "3", "label": "Operations"},
"function": {"id": "operations", "label": "Operations"}
}
]
}
+2 -2
View File
@@ -1,7 +1,7 @@
{
"offset": 0,
"limit": 100,
"totalFound": 2,
"limit": 2,
"totalFound": 3,
"content": [
{
"id": "744000151326819",
+25
View File
@@ -0,0 +1,25 @@
{
"total": 3,
"results": [
{
"id": 6125077,
"shortcode": "0A1B2C3D4E",
"title": "Maintenance Site Team Leader",
"remote": false,
"location": {
"country": "Ireland",
"countryCode": "IE",
"city": "Dublin",
"region": "County Dublin",
"display": "Dublin, County Dublin, Ireland"
},
"state": "published",
"published": "2026-09-28T00:00:00.000Z",
"type": "full",
"department": [
"Maintenance"
],
"workplace": "on_site"
}
]
}
+2 -1
View File
@@ -1,5 +1,6 @@
{
"total": 2,
"total": 3,
"nextPage": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0=",
"results": [
{
"id": 6125026,
+166 -1
View File
@@ -4,9 +4,12 @@ under `tests/fixtures/ats/`."""
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
@@ -160,13 +163,14 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
return (FIXTURES / "greenhouse.json").read_bytes()
monkeypatch.setattr(ats, "_get", fake_get)
rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
assert calls == [
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
]
assert [r["source_id"] for r in rows] == ["1001", "1002"]
assert complete # a host that does not page answers with the whole board
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
@@ -177,3 +181,164 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(ats._TooLarge):
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
# ----------------------------------------------------------------- paging
def _page(name: str) -> bytes:
return (FIXTURES / name).read_bytes()
def _status(code: int) -> httpx.HTTPStatusError:
request = httpx.Request("POST", "https://apply.workable.com/")
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
@pytest.fixture
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def test_workable_follows_next_page_to_the_stated_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable-2.json" if payload else "workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
assert complete
def test_smartrecruiters_pages_by_offset_to_total_found(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
urls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
urls.append(url)
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
assert complete
def test_a_board_that_ends_short_of_its_total_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""totalFound says 3; the second page is empty, so two were read."""
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
def fake_get(client, url, host, slug, *, method="GET", payload=None):
return empty if "offset=2" in url else _page("smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_a_repeated_row_does_not_count_twice_toward_the_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""A page that shifted under an offset repeats a role and misses one: three
rows read, two distinct, against a total of three."""
def fake_get(client, url, host, slug, *, method="GET", payload=None):
page = json.loads(_page("smartrecruiters.json"))
if "offset=2" in url:
page["content"] = page["content"][1:]
return json.dumps(page).encode()
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_the_page_cap_leaves_a_board_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable.json") # always another page
monkeypatch.setattr(ats, "MAX_PAGES", 3)
monkeypatch.setattr(ats, "_get", fake_get)
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert (len(calls), complete) == (3, False)
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
if payload:
raise _status(429)
return _page("workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
def test_a_429_on_the_first_page_is_a_failure(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
raise _status(429)
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(httpx.HTTPStatusError):
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
# -------------------------------------------------------- failing boards
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
def fake(client: object, target: str) -> object:
if isinstance(answer, Exception):
raise answer
return answer
monkeypatch.setattr(ats, "_fetch_rows", fake)
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}]})
source.state = {"ats:run": "run-1"}
return source
def test_a_failing_board_records_since_and_the_latest_reason(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, _status(404))
assert list(source.fetch()) == []
first = json.loads(source.state["ats:failing:greenhouse:gone"])
assert first["reason"] == "404 Not Found" and first["since"]
assert "ats:ok:greenhouse:gone" not in source.state
# A later failure keeps when the failures began and updates the reason.
source.state["ats:failing:greenhouse:gone"] = json.dumps(
first | {"since": "2026-09-01T00:00:00+00:00"})
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
later.state = source.state
list(later.fetch())
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
def test_an_answer_clears_the_failure_and_stamps_completeness(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, (_rows("greenhouse"), False))
source.state["ats:failing:greenhouse:gone"] = json.dumps(
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
assert len(list(source.fetch())) == 2
assert "ats:failing:greenhouse:gone" not in source.state
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
assert source.state["ats:complete:greenhouse:gone"] == ""
+93 -5
View File
@@ -4,8 +4,9 @@ touches the network."""
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import ClassVar
@@ -17,6 +18,7 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext # noqa: E402
from plugins import ats # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.jobs import pull # noqa: E402
from app.storage import Store # noqa: E402
@@ -126,6 +128,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
# (or its batch never got to it) and stamped nothing.
store.state["ats:run"] = "run-1"
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:complete:greenhouse:acme"] = "1"
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
class FakeTally:
@@ -133,8 +136,8 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1,
"closed": 1}
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 1, "closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -155,6 +158,91 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0,
"closed": 0}
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 0, "closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
# ---------------------------------------------- whole pulls, fetch stubbed
def _row(job_id: str) -> dict[str, object]:
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
"description": "", "remote": None, "salary": ""}
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
def fake(client: object, target: str) -> object:
if isinstance(answer, Exception):
raise answer
return answer
monkeypatch.setattr(ats, "_fetch_rows", fake)
def _pull(store: Store) -> dict[str, object]:
for batch in pull._boards(FakeConfig(), store): # type: ignore[arg-type]
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
class Tally:
results: ClassVar[list[dict[str, object]]] = []
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
def _open(store: Store) -> set[str]:
return {r.source_id for r in store.roles()}
@pytest.fixture
def acme(store: Store) -> Store:
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
board="workable")
return store
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
# A 429 on page two: page one's role is stored again, the other stays open.
_answer(monkeypatch, ([_row("1"), _row("3")], False))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
def test_a_failing_board_keeps_its_roles_before_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
_answer(monkeypatch, ValueError("gone"))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1"}
assert (result["failed"], result["closed"]) == (1, 0)
since, reason = pull.failing_boards(acme)["workable:acme"]
assert since and reason == "ValueError: gone"
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
_answer(monkeypatch, ValueError("404 for good"))
_pull(acme)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
timespec="seconds")
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
{"since": old, "reason": "404 Not Found"})
result = _pull(acme)
assert _open(acme) == set()
assert result["closed"] == 2
# Still failing, and still the date the failures began.
assert pull.failing_boards(acme)["workable:acme"][0] == old
_answer(monkeypatch, ([_row("2")], True))
_pull(acme)
assert _open(acme) == {"workable:acme:2"}
assert pull.failing_boards(acme) == {}
+55 -2
View File
@@ -4,6 +4,7 @@ searches. No network: Jobindex's feed is a trimmed copy of a real one
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -139,16 +140,68 @@ def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> N
return b"short" if params["page"] == 3 else b"full"
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setattr(search, "parse_jobindex",
lambda body: full[:5] if body == b"short" else full)
monkeypatch.setattr(search, "_jobindex_page",
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
# Stopped by the page limit with every page full: not complete.
pages.clear()
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert (pages, complete) == ([1, 2], False)
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
monkeypatch: pytest.MonkeyPatch) -> None:
"""A full page whose parser dropped one item still has 20 items, so the
search reads on instead of calling itself complete at 19 rows."""
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert not complete
def test_a_jobindex_answer_without_a_channel_raises() -> None:
with pytest.raises(ValueError, match="channel"):
search.parse_jobindex(b"<rss version='2.0'></rss>")
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
asked: list[int] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
page = int(url.rsplit("/", 1)[1])
asked.append(page)
start = (page - 1) * search.ADZUNA_PAGE
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
body: dict[str, Any] = {"results": results}
if count is not None:
body["count"] = count
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setenv("ADZUNA_APP_ID", "id")
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
return asked
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
def test_an_adzuna_search_is_complete_when_it_read_its_count(
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
_adzuna(monkeypatch, count)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (60, complete)
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
def test_provider_of() -> None:
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
assert search.provider_of("adzuna:berlin") == "adzuna"
+30 -2
View File
@@ -4,6 +4,7 @@ worker beyond starting and stopping it with the app."""
from __future__ import annotations
import json
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -395,7 +396,8 @@ def test_companies_lists_board_and_open_role_count(
[row] = [c for c in res.json() if c["key"] == "acme"]
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
"ats": "greenhouse", "careers_url": "", "country": "DK"}
"ats": "greenhouse", "careers_url": "", "country": "DK",
"failing_since": "", "failing_reason": ""}
[other] = [c for c in res.json() if c["key"] == "other"]
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
"", None, None, "")
@@ -404,6 +406,32 @@ def test_companies_lists_board_and_open_role_count(
"", None, "successfactors", "https://novo.example/careers")
def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,
capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch) -> None:
"""What the pull recorded, on `/api/companies` and beside `app companies
check`'s live result."""
import plugins.ats as ats
from app.cli import check_companies
store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme", ats="greenhouse")
store.state["ats:failing:greenhouse:acme"] = json.dumps(
{"since": "2026-09-01T06:00:00+00:00", "reason": "404 Not Found"})
[row] = client.get("/api/companies").json()
assert (row["failing_since"], row["failing_reason"]) == (
"2026-09-01T06:00:00+00:00", "404 Not Found")
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
assert check_companies(store) == 0
assert capsys.readouterr().out.splitlines() == [
"greenhouse:acme (greenhouse:acme): now 1 roles; "
"pull: failing since 2026-09-01: 404 Not Found",
"checked 1 boards, 0 dead, 1 failing in the pull",
]
def test_set_affinity_validates_updates_and_enqueues_a_score(
client: TestClient, store: Store, queue: Any) -> None:
store.add_company("Acme", key="acme")
@@ -909,7 +937,7 @@ def test_a_board_is_followed_only_if_it_answers(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: [{"source_id": "1"}])
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
assert res.status_code == 201
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"