Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
779b136757
commit
902089a5e2
14 files changed
+1608
-1492
No files matched your search
+49
-6
@@ -20,7 +20,9 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from app.models import Application # noqa: E402
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.jobs import cascade, pull # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
@@ -202,14 +204,16 @@ def _row(job_id: str) -> dict[str, object]:
|
||||
"description": "", "remote": None, "salary_text": ""}
|
||||
|
||||
|
||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
|
||||
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None:
|
||||
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
|
||||
with `filtered`, a read the board's filter narrowed."""
|
||||
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
rows, complete = answer # type: ignore[misc]
|
||||
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered)
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, object]:
|
||||
@@ -275,6 +279,45 @@ def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
|
||||
assert pull.failing_boards(acme) == {}
|
||||
|
||||
|
||||
def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
|
||||
_pull(acme)
|
||||
uid = {r.source_id: r.uid for r in acme.roles()}
|
||||
agent = acme.save_agent("Hunt")
|
||||
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
|
||||
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
|
||||
|
||||
_answer(monkeypatch, ([_row("1")], True), filtered=True)
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
||||
assert result["closed"] == 1
|
||||
assert acme.state["ats:filtered:workable:acme"] == "1"
|
||||
|
||||
# Read without the filter again, the board closes as before.
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
|
||||
|
||||
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
||||
board="workable", params={"countries": ["dk"], "remote": True})
|
||||
asked: list[object] = []
|
||||
|
||||
def fake(module: object, slug: str, get: object, ask: object, **kwargs: object
|
||||
) -> _runner.Listing:
|
||||
asked.append(ask)
|
||||
return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type]
|
||||
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
_pull(store)
|
||||
[ask] = asked
|
||||
declared = readers.HOSTS["workable"].READS.filters.declared()
|
||||
assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ------------------------------------------------------ one pull at a time
|
||||
|
||||
def _ago(**delta: float) -> str:
|
||||
|
||||
Reference in new issue
Block a user