Files
jobboard/tests/test_pull.py
T
WolfandClaude Opus 5.5 902089a5e2 Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole,
paged, one series per filter value unioned by id), completeness from READS,
the sweep of a board over the page cap across pulls, the 10% guard on a
filtered or empty="suspect" read, facets kept a day, the detail reads, the
raw an Item carries and the failure bookkeeping out of plugins/ats.py.
Ask comes from a subscription's params (ask_for, open for the reach), with
a declared key resolved into Ask.params["key"] and refused when missing.

The runtime robots.txt check goes (decided at build time, READS.robots).
plugins/search.py reads the providers of readers.PROVIDERS through the
runner and applies the saved pages param. A filtered board's close keeps
roles with a vote or an application. DETAIL_FIELDS gain places, teaser,
country and employment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 09:28:33 +02:00

433 lines
18 KiB
Python

"""The `pull` task: batching, the batch job's upsert, and the settle's
closure — with the `ats` plugin's own `fetch()` stubbed out, so nothing here
touches the network."""
from __future__ import annotations
import json
import sys
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import ClassVar
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext, Tally # noqa: E402
from app.models import Application # noqa: E402
from plugins import readers # noqa: E402
from plugins.readers import _runner # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.jobs import cascade, pull # noqa: E402
from app.storage import Store # noqa: E402
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[object]] = []
def _item(board: str, slug: str, job_id: str, company: str = "Acme") -> Item:
item = Item(
source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}",
title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": company, "place": "Berlin", "department": "",
"function": "engineering", "remote": None, "salary_text": "",
"board": board, "slug": slug})
item.source = "ats"
return item
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -> None:
for i in range(3):
store.add_subscription("ats", f"greenhouse:c{i}", label=f"C{i}",
company=f"C{i}", board="greenhouse")
off = store.add_subscription("ats", "greenhouse:off", label="Off",
company="Off", board="greenhouse")
store.update_subscription(off.id, fetch=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
assert sum(len(b["targets"]) for b in batches) == 3
run = store.state.get(pull.RUN, "")
assert run and {b["run"] for b in batches} == {run}
assert pull.running(store) == run
def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None:
assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type]
assert store.state.get(pull.RUN, "") == ""
def test_run_batch_upserts_every_item_the_plugin_yields(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "greenhouse:acme", label="Acme",
company="Acme", board="greenhouse")
def fake_fetch(self: ATSSource) -> object:
yield _item("greenhouse", "acme", "1")
yield _item("greenhouse", "acme", "2")
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
self.state["ats:ids:greenhouse:acme"] = '["1", "2"]'
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"], run="run-1")
assert result == {"roles": 2, "run": "run-1"}
assert store.state["ats:ok:greenhouse:acme"] == "run-1"
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"}
assert store.roles()[0].subscription_id is not None
def test_a_kill_stops_the_batch_before_the_next_board(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""`check()` before each board: a Kill pressed while the first is read
ends the job before the second is fetched, and keeps what the first
stored."""
for slug in ("acme", "beta"):
store.add_subscription("ats", f"greenhouse:{slug}", label=slug.title(),
company=slug.title(), board="greenhouse")
fetched: list[str] = []
killed = False
def fake_fetch(self: ATSSource) -> object:
nonlocal killed
[(target, _)] = self.targets()
fetched.append(target)
yield _item("greenhouse", target.partition(":")[2], "1")
killed = True
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
context = RunContext(is_cancelled=lambda: killed)
with pytest.raises(Cancelled):
pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme", "greenhouse:beta"],
context=context)
assert fetched == ["greenhouse:acme"]
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
def test_close_boards_closes_what_a_completed_board_no_longer_lists(
store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
company="Acme", board="greenhouse")
other = store.add_subscription("ats", "greenhouse:other", label="Other",
company="Other", board="greenhouse")
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
store.upsert_role(_item("greenhouse", "acme", "2"), sub)
store.upsert_role(_item("greenhouse", "other", "9", company="Other"), other)
# acme answered this run and now lists only "1"; other's board failed
# (or its batch never got to it) and stamped nothing.
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:complete:greenhouse:acme"] = "1"
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
class FakeTally:
results = ({"roles": 1, "run": "run-1"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 1, "closed": 1, "copies": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
company="Acme", board="greenhouse")
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
# A stale stamp from a run that is not this one — e.g. this run's batch
# for this board never finished.
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:ids:greenhouse:acme"] = "[]"
class FakeTally:
results = ({"roles": 0, "run": "run-2"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 0, "closed": 0, "copies": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
"""The settle's last step: a board's repeats of one job are one role,
and when the kept one closes the next copy is kept in its place."""
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
company="Acme", board="greenhouse")
for job_id in ("1", "2", "3"):
store.upsert_role(_item("greenhouse", "acme", job_id), sub)
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:complete:greenhouse:acme"] = "1"
store.state["ats:ids:greenhouse:acme"] = json.dumps(
["greenhouse:acme:1", "greenhouse:acme:2", "greenhouse:acme:3"])
class FakeTally:
results = ({"roles": 3, "run": "run-1"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result["copies"] == 2
[kept] = store.roles_page()
assert kept.copies == 2
gone = store.role(kept.uid).source_id # type: ignore[union-attr]
store.state["ats:ids:greenhouse:acme"] = json.dumps(
[f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone])
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert (result["closed"], result["copies"]) == (1, 1)
[after] = store.roles_page()
assert after.uid != kept.uid and after.copies == 1
# ---------------------------------------------- whole pulls, fetch stubbed
def _row(job_id: str) -> dict[str, object]:
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
"description": "", "remote": None, "salary_text": ""}
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None:
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
with `filtered`, a read the board's filter narrowed."""
def fake(*args: object, **kwargs: object) -> _runner.Listing:
if isinstance(answer, Exception):
raise answer
rows, complete = answer # type: ignore[misc]
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered)
monkeypatch.setattr(_runner, "read_listing", fake)
def _pull(store: Store) -> dict[str, object]:
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
def _open(store: Store) -> set[str]:
return {r.source_id for r in store.roles()}
@pytest.fixture
def acme(store: Store) -> Store:
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
board="workable")
return store
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
# A 429 on page two: page one's role is stored again, the other stays open.
_answer(monkeypatch, ([_row("1"), _row("3")], False))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
def test_a_failing_board_keeps_its_roles_before_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
_answer(monkeypatch, ValueError("gone"))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1"}
assert (result["failed"], result["closed"]) == (1, 0)
since, reason = pull.failing_boards(acme)["workable:acme"]
assert since and reason == "ValueError: gone"
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
_answer(monkeypatch, ValueError("404 for good"))
_pull(acme)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
timespec="seconds")
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
{"since": old, "reason": "404 Not Found"})
result = _pull(acme)
assert _open(acme) == set()
assert result["closed"] == 2
# Still failing, and still the date the failures began.
assert pull.failing_boards(acme)["workable:acme"][0] == old
_answer(monkeypatch, ([_row("2")], True))
_pull(acme)
assert _open(acme) == {"workable:acme:2"}
assert pull.failing_boards(acme) == {}
def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
_pull(acme)
uid = {r.source_id: r.uid for r in acme.roles()}
agent = acme.save_agent("Hunt")
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
_answer(monkeypatch, ([_row("1")], True), filtered=True)
result = _pull(acme)
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
assert result["closed"] == 1
assert acme.state["ats:filtered:workable:acme"] == "1"
# Read without the filter again, the board closes as before.
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
assert _open(acme) == {"workable:acme:1"}
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
board="workable", params={"countries": ["dk"], "remote": True})
asked: list[object] = []
def fake(module: object, slug: str, get: object, ask: object, **kwargs: object
) -> _runner.Listing:
asked.append(ask)
return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type]
monkeypatch.setattr(_runner, "read_listing", fake)
_pull(store)
[ask] = asked
declared = readers.HOSTS["workable"].READS.filters.declared()
assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined]
# ------------------------------------------------------ one pull at a time
def _ago(**delta: float) -> str:
return (datetime.now(UTC) - timedelta(**delta)).isoformat(timespec="microseconds")
def test_a_pull_during_a_running_one_is_refused_and_queues_nothing(acme: Store) -> None:
first = pull.claim_run(acme)
assert first and pull.running(acme) == first
asked: list[object] = []
said: list[object] = []
context = RunContext(on_task=lambda task: asked.append(task) or True,
on_notice=said.append)
result = cascade.run_pull(FakeConfig(), acme, context=context)
assert result == {"skipped": pull.ALREADY_RUNNING, "running": first}
assert asked == []
assert [n.code for n in said] == ["pull.running"] # type: ignore[attr-defined]
# A chain started without the job's claim (a test, a script) finds
# nothing to queue either, and the running run's marker stands.
assert pull._boards(FakeConfig(), acme) == [] # type: ignore[arg-type]
assert acme.state[pull.RUN] == first
def test_a_pull_starts_once_the_last_one_settled(acme: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
assert pull.running(acme) == ""
asked: list[object] = []
context = RunContext(on_task=lambda task: asked.append(task) or True)
result = cascade.run_pull(FakeConfig(), acme, context=context)
assert result["run"] == acme.state[pull.RUN] and len(asked) == 1
# The chain's finder reads under the job's claim instead of claiming again.
[batch] = asked[0].finder(FakeConfig(), acme) # type: ignore[attr-defined]
assert batch["run"] == result["run"]
def test_starters_claiming_at_once_give_one_run(tmp_path: Path) -> None:
"""Eight starters, each on its own connection to one file, as two
processes would be, released together: one claim wins."""
Store(tmp_path / "shared.db")
stores = [Store(tmp_path / "shared.db") for _ in range(8)]
barrier = threading.Barrier(len(stores))
def claim(store: Store) -> str:
barrier.wait()
return pull.claim_run(store)
with ThreadPoolExecutor(len(stores)) as pool:
claims = list(pool.map(claim, stores))
assert sum(bool(c) for c in claims) == 1
def test_an_abandoned_run_is_taken_over_after_the_threshold(acme: Store) -> None:
hours = pull.ABANDONED.total_seconds() / 3600
acme.state[pull.RUN] = _ago(hours=hours / 2)
assert pull.claim_run(acme) == ""
acme.state[pull.RUN] = _ago(hours=hours + 0.1)
taken = pull.claim_run(acme)
assert taken and acme.state[pull.RUN] == taken
def test_a_batch_stamps_with_the_marker_it_was_handed(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
[batch] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
acme.state[pull.RUN] = "a later run"
pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
assert acme.state["ats:ok:workable:acme"] == batch["run"]
def test_run_a_settle_does_not_close_on_run_b_ids(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""Run A read the board in full; run B, which took over, read it again and
listed less before A's settle ran. A's settle compares with A's marker, so
B's stamp and B's ids close nothing, and A's settle leaves B's claim held;
B's own settle is the one that closes."""
_answer(monkeypatch, ([_row("1"), _row("2")], True))
[a] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
a_result = pull.run_batch(FakeConfig(), acme, **a) # type: ignore[arg-type]
acme.state[pull.RUN] = _ago(hours=2) # A looks abandoned
_answer(monkeypatch, ([_row("1")], True))
[b] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
assert b["run"] != a["run"]
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
assert (result["failed"], result["closed"]) == (1, 0)
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
assert pull.running(acme) == b["run"]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(b_result,))) # type: ignore[arg-type]
assert result["closed"] == 1
assert _open(acme) == {"workable:acme:1"}
assert pull.running(acme) == ""
def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> None:
"""A batch that raised has no result, but its payload still names the run."""
run = pull.claim_run(acme)
acme.state["ats:ok:workable:acme"] = run
acme.state["ats:complete:workable:acme"] = "1"
acme.state["ats:ids:workable:acme"] = "[]"
failed = type("Job", (), {"payload": {"targets": ["workable:acme"], "run": run},
"result": None, "state": "failed"})()
context = RunContext(waited_for=(failed,))
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
assert result["boards"] == 1 and pull.running(acme) == ""