Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:<target>` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus <noreply@anthropic.com>
249 lines
9.5 KiB
Python
249 lines
9.5 KiB
Python
"""The `pull` task: batching, the batch job's upsert, and the settle's
|
|
closure — with the `ats` plugin's own `fetch()` stubbed out, so nothing here
|
|
touches the network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import ClassVar
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from desk.models import Item # noqa: E402
|
|
from pipeline import Cancelled, RunContext # noqa: E402
|
|
from plugins import ats # noqa: E402
|
|
from plugins.ats import ATSSource # noqa: E402
|
|
from app.jobs import pull # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
|
|
|
|
class FakeConfig:
|
|
sources: ClassVar[dict[str, object]] = {}
|
|
searches: ClassVar[list[object]] = []
|
|
|
|
|
|
def _item(board: str, slug: str, job_id: str) -> Item:
|
|
item = Item(
|
|
source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}",
|
|
title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": "Acme", "place": "Berlin", "department": "",
|
|
"function": "engineering", "remote": None, "salary": "",
|
|
"board": board, "slug": slug})
|
|
item.source = "ats"
|
|
return item
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -> None:
|
|
for i in range(3):
|
|
store.add_subscription("ats", f"greenhouse:c{i}", label=f"C{i}",
|
|
company=f"C{i}", board="greenhouse")
|
|
off = store.add_subscription("ats", "greenhouse:off", label="Off",
|
|
company="Off", board="greenhouse")
|
|
store.update_subscription(off.id, fetch=False)
|
|
|
|
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
|
assert sum(len(b["targets"]) for b in batches) == 3
|
|
assert store.state.get("ats:run", "") != ""
|
|
|
|
|
|
def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None:
|
|
assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type]
|
|
assert store.state.get("ats:run", "") == ""
|
|
|
|
|
|
def test_run_batch_upserts_every_item_the_plugin_yields(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
|
company="Acme", board="greenhouse")
|
|
|
|
def fake_fetch(self: ATSSource) -> object:
|
|
yield _item("greenhouse", "acme", "1")
|
|
yield _item("greenhouse", "acme", "2")
|
|
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
|
|
self.state["ats:ids:greenhouse:acme"] = '["1", "2"]'
|
|
|
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
|
store.state["ats:run"] = "run-1"
|
|
|
|
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"])
|
|
|
|
assert result == {"roles": 2}
|
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"}
|
|
assert store.roles()[0].subscription_id is not None
|
|
|
|
|
|
def test_a_kill_stops_the_batch_before_the_next_board(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""`check()` before each board: a Kill pressed while the first is read
|
|
ends the job before the second is fetched, and keeps what the first
|
|
stored."""
|
|
for slug in ("acme", "beta"):
|
|
store.add_subscription("ats", f"greenhouse:{slug}", label=slug.title(),
|
|
company=slug.title(), board="greenhouse")
|
|
fetched: list[str] = []
|
|
killed = False
|
|
|
|
def fake_fetch(self: ATSSource) -> object:
|
|
nonlocal killed
|
|
[(target, _)] = self.targets()
|
|
fetched.append(target)
|
|
yield _item("greenhouse", target.partition(":")[2], "1")
|
|
killed = True
|
|
|
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
|
context = RunContext(is_cancelled=lambda: killed)
|
|
|
|
with pytest.raises(Cancelled):
|
|
pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme", "greenhouse:beta"],
|
|
context=context)
|
|
|
|
assert fetched == ["greenhouse:acme"]
|
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
|
|
|
|
|
def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
|
store: Store) -> None:
|
|
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
|
company="Acme", board="greenhouse")
|
|
other = store.add_subscription("ats", "greenhouse:other", label="Other",
|
|
company="Other", board="greenhouse")
|
|
|
|
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
|
|
store.upsert_role(_item("greenhouse", "acme", "2"), sub)
|
|
store.upsert_role(_item("greenhouse", "other", "9"), other)
|
|
|
|
# acme answered this run and now lists only "1"; other's board failed
|
|
# (or its batch never got to it) and stamped nothing.
|
|
store.state["ats:run"] = "run-1"
|
|
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
|
store.state["ats:complete:greenhouse:acme"] = "1"
|
|
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
|
|
|
class FakeTally:
|
|
results = ({"roles": 1},)
|
|
|
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
|
|
|
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
|
"roles": 1, "closed": 1}
|
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
|
|
|
|
|
def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
|
store: Store) -> None:
|
|
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
|
company="Acme", board="greenhouse")
|
|
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
|
|
|
|
# A stale stamp from a run that is not this one — e.g. this run's batch
|
|
# for this board never finished.
|
|
store.state["ats:run"] = "run-2"
|
|
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
|
store.state["ats:ids:greenhouse:acme"] = "[]"
|
|
|
|
class FakeTally:
|
|
results: tuple[dict[str, int], ...] = ()
|
|
|
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
|
|
|
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
|
"roles": 0, "closed": 0}
|
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
|
|
|
|
|
# ---------------------------------------------- whole pulls, fetch stubbed
|
|
|
|
def _row(job_id: str) -> dict[str, object]:
|
|
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
|
|
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
|
|
"description": "", "remote": None, "salary": ""}
|
|
|
|
|
|
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
|
|
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
|
|
def fake(client: object, target: str) -> object:
|
|
if isinstance(answer, Exception):
|
|
raise answer
|
|
return answer
|
|
|
|
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
|
|
|
|
|
def _pull(store: Store) -> dict[str, object]:
|
|
for batch in pull._boards(FakeConfig(), store): # type: ignore[arg-type]
|
|
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
|
|
|
class Tally:
|
|
results: ClassVar[list[dict[str, object]]] = []
|
|
|
|
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
|
|
|
|
|
|
def _open(store: Store) -> set[str]:
|
|
return {r.source_id for r in store.roles()}
|
|
|
|
|
|
@pytest.fixture
|
|
def acme(store: Store) -> Store:
|
|
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
|
board="workable")
|
|
return store
|
|
|
|
|
|
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
|
_pull(acme)
|
|
# A 429 on page two: page one's role is stored again, the other stays open.
|
|
_answer(monkeypatch, ([_row("1"), _row("3")], False))
|
|
result = _pull(acme)
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
|
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
|
|
|
|
|
|
def test_a_failing_board_keeps_its_roles_before_stale_days(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row("1")], True))
|
|
_pull(acme)
|
|
_answer(monkeypatch, ValueError("gone"))
|
|
result = _pull(acme)
|
|
assert _open(acme) == {"workable:acme:1"}
|
|
assert (result["failed"], result["closed"]) == (1, 0)
|
|
since, reason = pull.failing_boards(acme)["workable:acme"]
|
|
assert since and reason == "ValueError: gone"
|
|
|
|
|
|
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
|
_pull(acme)
|
|
_answer(monkeypatch, ValueError("404 for good"))
|
|
_pull(acme)
|
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
|
timespec="seconds")
|
|
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
|
{"since": old, "reason": "404 Not Found"})
|
|
|
|
result = _pull(acme)
|
|
assert _open(acme) == set()
|
|
assert result["closed"] == 2
|
|
# Still failing, and still the date the failures began.
|
|
assert pull.failing_boards(acme)["workable:acme"][0] == old
|
|
|
|
_answer(monkeypatch, ([_row("2")], True))
|
|
_pull(acme)
|
|
assert _open(acme) == {"workable:acme:2"}
|
|
assert pull.failing_boards(acme) == {}
|