Runner: readers read from their declaration, ats and search thin over it

plugins/readers/_runner.py takes the guarded fetch, the listing (whole,
paged, one series per filter value unioned by id), completeness from READS,
the sweep of a board over the page cap across pulls, the 10% guard on a
filtered or empty="suspect" read, facets kept a day, the detail reads, the
raw an Item carries and the failure bookkeeping out of plugins/ats.py.
Ask comes from a subscription's params (ask_for, open for the reach), with
a declared key resolved into Ask.params["key"] and refused when missing.

The runtime robots.txt check goes (decided at build time, READS.robots).
plugins/search.py reads the providers of readers.PROVIDERS through the
runner and applies the saved pages param. A filtered board's close keeps
roles with a vote or an application. DETAIL_FIELDS gain places, teaser,
country and employment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:28:33 +02:00
1 parent 779b136757
commit 902089a5e2
14 files changed
+1608 -1492

No files matched your search

+49 -6
View File
@@ -20,7 +20,9 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext, Tally # noqa: E402
from plugins import ats # noqa: E402
from app.models import Application # noqa: E402
from plugins import readers # noqa: E402
from plugins.readers import _runner # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.jobs import cascade, pull # noqa: E402
from app.storage import Store # noqa: E402
@@ -202,14 +204,16 @@ def _row(job_id: str) -> dict[str, object]:
"description": "", "remote": None, "salary_text": ""}
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
def fake(client: object, target: str) -> object:
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None:
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
with `filtered`, a read the board's filter narrowed."""
def fake(*args: object, **kwargs: object) -> _runner.Listing:
if isinstance(answer, Exception):
raise answer
return answer
rows, complete = answer # type: ignore[misc]
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered)
monkeypatch.setattr(ats, "_fetch_rows", fake)
monkeypatch.setattr(_runner, "read_listing", fake)
def _pull(store: Store) -> dict[str, object]:
@@ -275,6 +279,45 @@ def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
assert pull.failing_boards(acme) == {}
def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
_pull(acme)
uid = {r.source_id: r.uid for r in acme.roles()}
agent = acme.save_agent("Hunt")
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
_answer(monkeypatch, ([_row("1")], True), filtered=True)
result = _pull(acme)
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
assert result["closed"] == 1
assert acme.state["ats:filtered:workable:acme"] == "1"
# Read without the filter again, the board closes as before.
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
assert _open(acme) == {"workable:acme:1"}
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
board="workable", params={"countries": ["dk"], "remote": True})
asked: list[object] = []
def fake(module: object, slug: str, get: object, ask: object, **kwargs: object
) -> _runner.Listing:
asked.append(ask)
return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type]
monkeypatch.setattr(_runner, "read_listing", fake)
_pull(store)
[ask] = asked
declared = readers.HOSTS["workable"].READS.filters.declared()
assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined]
# ------------------------------------------------------ one pull at a time
def _ago(**delta: float) -> str: