Paged boards are read to their stated total, and a board failing for 21 days closes

Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:02:49 +02:00
1 parent 0558ef19ed
commit c7da12b93d
18 files changed
+695 -91

No files matched your search

+93 -5
View File
@@ -4,8 +4,9 @@ touches the network."""
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import ClassVar
@@ -17,6 +18,7 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext # noqa: E402
from plugins import ats # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.jobs import pull # noqa: E402
from app.storage import Store # noqa: E402
@@ -126,6 +128,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
# (or its batch never got to it) and stamped nothing.
store.state["ats:run"] = "run-1"
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:complete:greenhouse:acme"] = "1"
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
class FakeTally:
@@ -133,8 +136,8 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1,
"closed": 1}
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 1, "closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -155,6 +158,91 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0,
"closed": 0}
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 0, "closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
# ---------------------------------------------- whole pulls, fetch stubbed
def _row(job_id: str) -> dict[str, object]:
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
"description": "", "remote": None, "salary": ""}
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
def fake(client: object, target: str) -> object:
if isinstance(answer, Exception):
raise answer
return answer
monkeypatch.setattr(ats, "_fetch_rows", fake)
def _pull(store: Store) -> dict[str, object]:
for batch in pull._boards(FakeConfig(), store): # type: ignore[arg-type]
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
class Tally:
results: ClassVar[list[dict[str, object]]] = []
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
def _open(store: Store) -> set[str]:
return {r.source_id for r in store.roles()}
@pytest.fixture
def acme(store: Store) -> Store:
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
board="workable")
return store
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
# A 429 on page two: page one's role is stored again, the other stays open.
_answer(monkeypatch, ([_row("1"), _row("3")], False))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
def test_a_failing_board_keeps_its_roles_before_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
_answer(monkeypatch, ValueError("gone"))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1"}
assert (result["failed"], result["closed"]) == (1, 0)
since, reason = pull.failing_boards(acme)["workable:acme"]
assert since and reason == "ValueError: gone"
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
_answer(monkeypatch, ValueError("404 for good"))
_pull(acme)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
timespec="seconds")
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
{"since": old, "reason": "404 Not Found"})
result = _pull(acme)
assert _open(acme) == set()
assert result["closed"] == 2
# Still failing, and still the date the failures began.
assert pull.failing_boards(acme)["workable:acme"][0] == old
_answer(monkeypatch, ([_row("2")], True))
_pull(acme)
assert _open(acme) == {"workable:acme:2"}
assert pull.failing_boards(acme) == {}