From 9c8a6107877fad5fc80cab4f6416d3d075e2b3c2 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 15:53:47 +0200 Subject: [PATCH] Guard each target in a batch, close remote-unasked and narrowed roles by age, settle failing searches, and raise one notice per settle naming what needs a look Co-Authored-By: Claude Fable 5.1 --- app/jobs/cascade.py | 12 +- app/jobs/pull.py | 389 +++++++++++++++++++++++++++++++------------ tests/test_pull.py | 237 +++++++++++++++++++++++++- tests/test_search.py | 53 ++++++ 4 files changed, 576 insertions(+), 115 deletions(-) diff --git a/app/jobs/cascade.py b/app/jobs/cascade.py index c430545..3b389e1 100644 --- a/app/jobs/cascade.py +++ b/app/jobs/cascade.py @@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str Under `NULL` there is no queue to take the chain, so this raises rather than report a pull that never happened; `app job pull` claims the run and - starts the chain on a queue of its own instead of calling this. + starts the chain on a queue of its own instead of calling this. Anything + that raises once the run is claimed, building the chain or handing it + over, releases the claim first, or the next pull would wait + `pull.ABANDONED` for a run that never started. """ run = pull.claim_run(store) if not run: since = pull.running(store) context.notice(already_running(since)) return {"skipped": pull.ALREADY_RUNNING, "running": since} - if not context.task(pull_chain(run)): + try: + started = context.task(pull_chain(run)) + except Exception: + pull.release_run(store, run) + raise + if not started: pull.release_run(store, run) raise RuntimeError("no queue to start the pull cascade on") return {"started": " > ".join(CASCADES["pull"]), "run": run} diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 09368d2..cec1eea 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see `plugins/readers/_runner.py`). A board read with a filter (`ats:filtered:`, from the agents' reach, -`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a -person acted on: those with a vote or an application (`_close_unlisted`). A -filtered read, or one of a reader that declares `empty="suspect"`, that lists -under a tenth of what the same read listed last time is recorded by the -runner as a failure, not as a list to close against. +`app.jobs.reach`, or its subscription's `params`) lists only the roles in +reach, so its close keeps open the roles a person acted on: those with a vote +or an application (`_close_unlisted`). When the reach wanted remote roles and +the reader could not ask for them (`ats:remote_unasked:`), the read +says nothing of the remote roles either, so they stay open too and close only +once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read +that lists under a tenth of what the same read listed last time is recorded +by the runner as a failure, not as a list to close against (`_runner.read`). A board swept across pulls (`ats:swept:`, a complete read that finished a sweep an earlier pull began) closes a role only when two finished sweeps in a @@ -33,14 +36,21 @@ not swept clears them. A read narrowed by the agents' fetch terms (`ats:narrowed:`) lists the roles matching a word, which says nothing of the roles it did not list, -so it closes nothing; only an unfiltered or a place-filtered complete read -closes. +so it closes by list nothing: a role of such a board closes once the board +has not listed it for `STALE_DAYS`, the rule searches have. A search closes the same way when it listed everything that matches (see `plugins/search.py` for how each provider knows). One that did not closes its -roles only once no search has seen them for `STALE_DAYS`. A board -that has failed on every fetch for `STALE_DAYS` (`ats:failing:`) -closes all of its roles, and its next answer reopens what it lists. +roles only once no search has seen them for `STALE_DAYS`. A board or a +search that has failed on every fetch for `STALE_DAYS` (`ats:failing:`, +`search:failing:`) closes all of its roles, and its next answer +reopens what it lists. A target that raises in its batch after its read, a +row that does not store, is recorded as failing the same way and the batch +goes on with the next one (`run_batch`). + +The settle says what needs a look in one notice (`settled`): the targets +that did not answer or were read incomplete, and the boards that closed more +than `CLOSE_NOTICE` roles or more than half of theirs. One pull at a time. A run is claimed in the database (`claim_run`: `pull:run` set only while the previous run is settled, in one statement), so the @@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta from functools import partial from typing import Any -from pipeline import NULL, RunContext, Tally, Task, job +from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job from app.config import Config from app.jobs.reach import reaches @@ -91,6 +101,19 @@ STALE_DAYS = 21 #: `ats:failing:` -> `{"since", "reason"}`, removed on its next answer. FAILING = "ats:failing:" +#: The same for a search, kept by the `search` plugin. +SEARCH_FAILING = "search:failing:" + +#: A board that closes more roles than this in one settle is named in the +#: settle's notice, and so is one that closes more than half of its open +#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing +#: one is an ordinary day. +CLOSE_NOTICE = 50 +CLOSE_SHARE_MIN = 10 + +#: How many targets a notice names in its text; its detail names them all. +NOTICE_NAMES = 10 + #: The `state` key that records the starter searches were imported. SEARCHES_SEEDED = "searches:seeded" @@ -142,27 +165,38 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]] `close_boards` compares against. `run` is a marker the caller claimed already (the `pull` job, `app job pull`); without one the finder claims its own, and queues nothing when another run is in flight. A run with - nothing to read is settled here, since no settle will see a batch of it.""" - seed_searches(config, store) - subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES} - targets = [(source, [s.target for s in subs[source]]) for source in SOURCES] - if not any(found for _, found in targets): - if run: - release_run(store, run) - return [] - run = run or claim_run(store) - if not run: - log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING, - running(store)) - return [] - # The agents' reach, once per run, each board's in its batch (`app.jobs.reach`). - reach_of = reaches(store, subs["ats"]) - batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run} - for source, found in targets for i in range(0, len(found), BATCH_SIZE)] - for batch in batches: - if batch["source"] == "ats": - batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of} - return batches + nothing to read is settled here, since no settle will see a batch of it, + and so is a run whose finder raises: no batch and no settle would follow, + and the next pull would wait `ABANDONED` for a claim nobody holds.""" + held = run + try: + seed_searches(config, store) + subs = {source: store.subscriptions(source=source, fetchable=True) + for source in SOURCES} + targets = [(source, [s.target for s in subs[source]]) for source in SOURCES] + if not any(found for _, found in targets): + if run: + release_run(store, run) + return [] + run = run or claim_run(store) + if not run: + log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING, + running(store)) + return [] + held = run + # The agents' reach, once per run, each board's in its batch (`app.jobs.reach`). + reach_of = reaches(store, subs["ats"]) + batches: list[dict[str, Any]] = [ + {"targets": found[i:i + BATCH_SIZE], "source": source, "run": run} + for source, found in targets for i in range(0, len(found), BATCH_SIZE)] + for batch in batches: + if batch["source"] == "ats": + batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of} + return batches + except Exception: + if held: + release_run(store, held) + raise @job("pull_batch", "Reading boards", resource="http") @@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = plugin as its `run` option, so its stamps belong to that run whatever the stored marker says by the time the batch runs. `reach` is `target ->` the reach the run computed for a board (`app.jobs.reach`), handed to the - plugin as its `reach` option. + plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after + which the runner's 10% guard accepts the answer it kept refusing. + + Each target is guarded on its own. A read that fails is the plugin's to + record; anything raised after it, a row `upsert_role` refuses or a bug + past the read, is recorded here under `:failing:` with + its reason, and the batch goes on with the next target. Otherwise one + bad row would cost the targets after it in the batch their read on every + pull, since batches are stable, and leave no trace on any of them. A + Kill (`Cancelled`) still ends the batch. """ from plugins.ats import ATSSource + from plugins.readers import _runner from plugins.search import SearchSource # Fetchable now, not when the pull was queued: a board switched off on the @@ -200,24 +244,39 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = if sub is None: continue context.check() - options = dict(config.sources.get(source) or {}) | {"run": run} - plugin: ATSSource | SearchSource - if source == "search": - options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}] - plugin = SearchSource(options) - else: - # A board's params are its filter; a reader's key is kept with - # the searches' (`sources.search.`). - options["boards"] = [{"url": target, "label": sub.company or sub.label, - "params": sub.params}] - options["keys"] = dict(config.sources.get("search") or {}) - options["reach"] = (reach or {}).get(target) - plugin = ATSSource(options) - plugin.state = store.state - for item in plugin.fetch(): - item.source = source - store.upsert_role(item, sub) - stored += 1 + options = dict(config.sources.get(source) or {}) | {"run": run, + "stale_days": STALE_DAYS} + items: Any = None + try: + plugin: ATSSource | SearchSource + if source == "search": + options["searches"] = [{"url": target, "label": sub.label, + "params": sub.params}] + plugin = SearchSource(options) + else: + # A board's params are its filter; a reader's key is kept with + # the searches' (`sources.search.`). + options["boards"] = [{"url": target, "label": sub.company or sub.label, + "params": sub.params}] + options["keys"] = dict(config.sources.get("search") or {}) + options["reach"] = (reach or {}).get(target) + plugin = ATSSource(options) + plugin.state = store.state + items = plugin.fetch() + for item in items: + item.source = source + store.upsert_role(item, sub) + stored += 1 + except Cancelled: + raise + except Exception as exc: + log.warning("pull: %s failed after its read (%s); on to the next target", + target, exc, exc_info=True) + _runner.record_failure(store.state, f"{source}:failing:{target}", exc) + finally: + # The plugin's generator holds its HTTP client until it is closed. + if items is not None and hasattr(items, "close"): + items.close() return {"roles": stored, "run": run} @@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str: def close_boards(config: Config, store: Store, tally: Tally, *, context: RunContext = NULL) -> dict[str, Any]: """The run's settle: close every board this run answered in full and - stopped listing a role from, and tally what happened. + stopped listing a role from, tally what happened, and say what needs a + look. Reads `self.state` rather than the batch jobs' own results, because a `pull_batch` job that raised partway through still leaves the boards it @@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *, `ats:ok:` was not written during *this* run — it failed, or its whole batch job never got to it — is never closed by its list, and a board that answered incompletely is not either. A search closes by its list only - when that list was complete, and otherwise by `STALE_DAYS`. A board that - has failed on every fetch since `STALE_DAYS` ago closes all its roles. + when that list was complete, and otherwise by `STALE_DAYS`. A board or a + search that has failed on every fetch since `STALE_DAYS` ago closes all + its roles (`failing_boards`). How a read that answered closes is + `_close_read`'s. The stamps are compared with this run's own marker, the one its batches carried, never with the stored `pull:run`, which a later run may have replaced. A settle with no batch behind it (a refused pull, nothing to - read) closes nothing by list or by age. Last, the run is marked settled, - which lets the next pull start. + read) closes nothing by list or by age and says nothing. Last, the run is + marked settled, which lets the next pull start. A board or search switched off while its batch was in flight had its roles reopened by that batch, after the switch closed them, so every - switched-off one is closed again here. + switched-off one is closed again here; those closes are asked for and not + named in the notice. + + The counts go into the result (`unanswered` counts boards and searches + whose stamp is not this run's), and one notice (`settled`) names every + target unanswered, every board read incomplete other than one mid-sweep, + and every board that closed more than `CLOSE_NOTICE` roles or more than + half the open roles it had, when there is any. The result alone was + seen by nobody: a settle closed 12,588 roles with nothing on screen. The `prep` stage that follows maps what this run stored and folds the copies (`Store.rebuild_dupes`), so a closed kept role hands its group on and the roles this run added join theirs. """ - answered = searched = failed = incomplete = closed = 0 + answered = searched = incomplete = closed = 0 + unanswered: list[str] = [] + partial: list[str] = [] + many: list[tuple[str, int, int]] = [] stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds") failing = failing_boards(store) run_marker = _run_of(tally, context) + open_before = _open_counts(store) if run_marker else {} for source in SOURCES: for sub in store.subscriptions(source=source): if not (sub.subscribed and sub.fetch): @@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *, for sub in store.subscriptions(source=source, fetchable=True): stamp = store.state.get(f"{source}:ok:{sub.target}", "") if not (stamp and stamp == run_marker): - failed += 1 - since = failing.get(sub.target, ("", ""))[0] if source == "ats" else "" - if since and since < stale: - closed += store.close_subscription(sub.id) - continue - if source == "search": - searched += 1 + unanswered.append(sub.target) + since = failing.get(sub.target, ("", ""))[0] + shut = store.close_subscription(sub.id) if since and since < stale else 0 else: - answered += 1 - if not store.state.get(f"{source}:complete:{sub.target}", ""): - if source == "ats": - incomplete += 1 - continue - if store.state.get(f"{source}:narrowed:{sub.target}", ""): - continue - seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) - is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", "")) - if store.state.get(f"{source}:swept:{sub.target}", ""): - closed += _close_swept(store, source, sub, seen_ids, is_filtered) - continue - store.state.pop(f"{source}:missing:{sub.target}", None) - if is_filtered: - closed += _close_unlisted(store, source, seen_ids, sub.id) - else: - closed += store.mark_closed(source, seen_ids, sub.id) + if source == "search": + searched += 1 + else: + answered += 1 + read = _close_read(store, source, sub, stale) + if read is None: + if source == "ats": + incomplete += 1 + if not store.state.get(f"ats:sweep:{sub.target}"): + partial.append(sub.target) + continue + shut = read + closed += shut + before = open_before.get(sub.id, 0) + if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before): + many.append((sub.target, shut, before)) if searched: closed += store.close_unseen("search", stale) if run_marker: + if unanswered or partial or many: + context.notice(settled(unanswered, partial, many, failing, + boards=answered, searches=searched, closed=closed)) release_run(store, run_marker) roles = sum(int(result.get("roles", 0)) for result in tally.results) - return {"boards": answered, "searches": searched, "failed": failed, + return {"boards": answered, "searches": searched, "unanswered": len(unanswered), "incomplete": incomplete, "roles": roles, "closed": closed} -def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int: +def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None: + """Close what one target's read of this run says has closed, and return + how many; None when the read was incomplete, which closes nothing. + + A read narrowed by terms closes only the roles it has not listed for + `STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what + two sweeps in a row missed (`_close_swept`); any other complete read + clears that record and closes what it did not list, all of it unfiltered + (`Store.mark_closed`), filtered sparing the roles acted on + (`_close_unlisted`). A filtered read that could not ask for remote roles + (`remote_unasked`) spares the remote ones as well, and closes those by + age instead.""" + state, target = store.state, sub.target + if not state.get(f"{source}:complete:{target}", ""): + return None + seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]")) + if state.get(f"{source}:narrowed:{target}", ""): + return _close_aged(store, source, sub.id, seen_ids, stale) + is_filtered = bool(state.get(f"{source}:filtered:{target}", "")) + spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", "")) + aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True) + if spare_remote else 0) + if state.get(f"{source}:swept:{target}", ""): + return aged + _close_swept(store, source, sub, seen_ids, is_filtered, + spare_remote=spare_remote) + state.pop(f"{source}:missing:{target}", None) + if is_filtered: + return aged + _close_unlisted(store, source, seen_ids, sub.id, + spare_remote=spare_remote) + return store.mark_closed(source, seen_ids, sub.id) + + +def _open_counts(store: Store) -> dict[int, int]: + """`subscription id -> open roles`, before the settle closes any, for the + share a board closed.""" + with store.transaction() as conn: + return {int(row[0]): int(row[1]) for row in conn.execute( + """SELECT subscription_id, count(*) FROM roles + WHERE closed_at IS NULL AND subscription_id IS NOT NULL + GROUP BY subscription_id""")} + + +def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]], + failing: dict[str, tuple[str, str]], *, boards: int, searches: int, + closed: int) -> Notice: + """The notice of a settle that has something to look at: the targets + that did not answer (with the reason their failing record gives), the + boards read incomplete, and the boards that closed many roles, each + as `closed of open`, with the totals. The text names the first + `NOTICE_NAMES` of each, the detail all of them, one a line.""" + def named(target: str) -> str: + reason = failing.get(target, ("", ""))[1] + return f"{target} ({reason})" if reason else target + + groups = [("Not answering", [named(t) for t in unanswered]), + ("Read incomplete", list(incomplete)), + ("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])] + text = [f"The pull read {boards} boards and {searches} searches and closed " + f"{closed:,} roles."] + detail: list[str] = [] + for title, names in groups: + if not names: + continue + shown = ", ".join(names[:NOTICE_NAMES]) + more = len(names) - NOTICE_NAMES + text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0 + else ".")) + detail += [f"{title}: {name}" for name in names] + return Notice(code="pull.settled", level="warning", kind="expected", key="pull", + text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull") + + +#: The roles a partial read never closes by its list: one with a vote (any, +#: neutral included) or an application (which holds the notes). +_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes) + AND uid NOT IN (SELECT role_uid FROM applications + WHERE role_uid IS NOT NULL)""" + +#: The remote roles a filtered read that could not ask for them leaves alone. +_NOT_REMOTE = " AND remote != 'remote'" + + +def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int, + *, spare_remote: bool = False) -> int: """`Store.mark_closed` for a filtered read: close every open role of the - board it did not list, except a role someone acted on, one with a vote - (any, neutral included) or an application (which holds the notes). A + board it did not list, except a role someone acted on (`_ACTED_ON`). A filtered read cannot tell a role gone from a role outside its filter, and a role acted on stays until the board is read without the filter or - fails for `STALE_DAYS`.""" + fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as + well, since the read could not ask for them.""" with store.transaction() as conn: return conn.execute( - """UPDATE roles SET closed_at = ? - WHERE source = ? AND subscription_id = ? AND closed_at IS NULL - AND source_id NOT IN (SELECT value FROM json_each(?)) - AND uid NOT IN (SELECT uid FROM role_votes) - AND uid NOT IN (SELECT role_uid FROM applications - WHERE role_uid IS NOT NULL)""", + f"""UPDATE roles SET closed_at = ? + WHERE source = ? AND subscription_id = ? AND closed_at IS NULL + AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}""" + + (_NOT_REMOTE if spare_remote else ""), (store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount +def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str], + stale: str, *, remote_only: bool = False) -> int: + """Close the open roles of one board that its read did not list and that + no read has listed since `stale` (`roles.last_seen`, which `upsert_role` + writes on every listing): the rule for what a read cannot speak for. A + read narrowed by terms closes all such roles, as a search does; with + `remote_only`, a filtered read that could not ask for remote closes only + its remote roles so, sparing those acted on as its list does.""" + where = " AND remote = 'remote'" + _ACTED_ON if remote_only else "" + with store.transaction() as conn: + return conn.execute( + f"""UPDATE roles SET closed_at = ? + WHERE source = ? AND subscription_id = ? AND closed_at IS NULL + AND last_seen < ? + AND source_id NOT IN (SELECT value FROM json_each(?)){where}""", + (store.now(), source, subscription_id, stale, + json.dumps(list(seen_ids)))).rowcount + + def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str], - filtered: bool) -> int: + filtered: bool, *, spare_remote: bool = False) -> int: """The close of a board whose complete read finished a sweep across pulls: its open roles missing from `seen_ids` that also are in `:missing:`, the ones the previous finished sweep missed. @@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str], missed may only have shifted back past the cursor when roles closed between pulls, so one sweep is not evidence that it is gone. A filtered read keeps its exceptions as `_close_unlisted` does: a role acted on - stays open, and stays in `missing`.""" + stays open, and stays in `missing`, and so does a remote role with + `spare_remote`.""" key = f"{source}:missing:{sub.target}" try: earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")} except (TypeError, ValueError): earlier = set() - acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes) - AND uid NOT IN (SELECT role_uid FROM applications - WHERE role_uid IS NOT NULL)""" if filtered else "" + acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "") with store.transaction() as conn: missed = [row[0] for row in conn.execute( """SELECT source_id FROM roles @@ -365,16 +536,20 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str], def failing_boards(store: Store) -> dict[str, tuple[str, str]]: - """`target -> (since, reason)` for every `ats` board whose last fetch - failed: when the failures began, and what the latest one said.""" + """`target -> (since, reason)` for every `ats` board and every search + whose last fetch failed: when the failures began, and what the latest + one said. No board host is also a search provider, so a board's target + (`host:slug`) and a search's (`provider:name`) never collide in one + dict.""" out: dict[str, tuple[str, str]] = {} - for key, value in store.state_items(FAILING).items(): - try: - record = json.loads(value) - except ValueError: - continue - out[key[len(FAILING):]] = (str(record.get("since") or ""), - str(record.get("reason") or "")) + for prefix in (FAILING, SEARCH_FAILING): + for key, value in store.state_items(prefix).items(): + try: + record = json.loads(value) + except ValueError: + continue + out[key[len(prefix):]] = (str(record.get("since") or ""), + str(record.get("reason") or "")) return out diff --git a/tests/test_pull.py b/tests/test_pull.py index 6a94718..1ee5e8f 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -7,6 +7,7 @@ from __future__ import annotations import json import sys import threading +import types from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from pathlib import Path @@ -21,8 +22,8 @@ if str(ROOT) not in sys.path: from desk.models import Item # noqa: E402 from pipeline import Cancelled, RunContext, Tally # noqa: E402 from app.models import Application # noqa: E402 -from plugins import readers # noqa: E402 -from plugins.readers import _runner # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import Filters, Reads, _runner # noqa: E402 from plugins.ats import ATSSource # noqa: E402 from app import prep # noqa: E402 from app.jobs import cascade, pull # noqa: E402 @@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0, + assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0, "roles": 1, "closed": 1} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} @@ -163,7 +164,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0, + assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0, "roles": 0, "closed": 0} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} @@ -256,7 +257,7 @@ def test_a_failing_board_keeps_its_roles_before_stale_days( _answer(monkeypatch, ValueError("gone")) result = _pull(acme) assert _open(acme) == {"workable:acme:1"} - assert (result["failed"], result["closed"]) == (1, 0) + assert (result["unanswered"], result["closed"]) == (1, 0) since, reason = pull.failing_boards(acme)["workable:acme"] assert since and reason == "ValueError: gone" @@ -486,7 +487,7 @@ def test_run_a_settle_does_not_close_on_run_b_ids( b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type] - assert (result["failed"], result["closed"]) == (1, 0) + assert (result["unanswered"], result["closed"]) == (1, 0) assert _open(acme) == {"workable:acme:1", "workable:acme:2"} assert pull.running(acme) == b["run"] @@ -507,3 +508,227 @@ def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> No context = RunContext(waited_for=(failed,)) result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type] assert result["boards"] == 1 and pull.running(acme) == "" + + +# -------------------------------------------------------- losses, guarded + +def _days_ago(days: float) -> str: + return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds") + + +def _set(store: Store, column: str, value: str, *source_ids: str) -> None: + with store.transaction() as conn: + conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?", + [(value, i) for i in source_ids]) + + +@pytest.fixture +def big(store: Store) -> Store: + store.add_subscription("ats", "greenhouse:big", label="Big", company="Big", + board="greenhouse") + return store + + +def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails( + big: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(str(n)) for n in range(40)], True)) + _pull(big) + _answer(monkeypatch, ([], True)) + result = _pull(big) + assert len(_open(big)) == 40 + assert (result["unanswered"], result["closed"]) == (1, 0) + assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1] + + +def test_a_whole_read_of_nothing_after_five_closes_the_board( + big: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(str(n)) for n in range(5)], True)) + _pull(big) + _answer(monkeypatch, ([], True)) + assert _pull(big)["closed"] == 5 + assert _open(big) == set() + + +def test_a_suspect_answer_is_accepted_after_stale_days( + big: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(str(n)) for n in range(40)], True)) + _pull(big) + _answer(monkeypatch, ([_row("1")], True)) + _pull(big) + assert len(_open(big)) == 40 + record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"]) + big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps( + record | {"since": _days_ago(pull.STALE_DAYS + 1)}) + result = _pull(big) + assert _open(big) == {"greenhouse:big:1"} + assert (result["unanswered"], result["closed"]) == (0, 39) + assert pull.failing_boards(big) == {} + assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1 + + +def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]: + """A whole-listing host `fake` that filters by country and, per `remote`, + on remote; its feed lists the ids in the returned list.""" + listed: list[str] = [] + module: object = types.ModuleType("fake") + module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined] + module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined] + module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined] + module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined] + module.READS = Reads( # type: ignore[attr-defined] + scope="board", listing="whole", text="listing", complete="whole", places="one", + robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type] + module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined] + module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined] + monkeypatch.setitem(readers.HOSTS, "fake", module) + monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps( + listed).encode()) + return listed + + +def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + listed = _remote_host(monkeypatch, remote="none") + store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake") + reach = {"fake:acme": {"countries": ["DK"], "remote": True}} + + def pull_once() -> dict[str, object]: + run = pull.claim_run(store) + result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type] + reach=reach) + return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type] + + listed[:] = ["1", "2", "3", "4"] + pull_once() + _set(store, "remote", "remote", "fake:acme:3", "fake:acme:4") + listed[:] = ["1"] + assert pull_once()["closed"] == 1 + assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"} + assert store.state["ats:remote_unasked:fake:acme"] == "1" + + _set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4") + assert pull_once()["closed"] == 1 + assert _open(store) == {"fake:acme:1", "fake:acme:3"} + + +def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(i) for i in "123"], True)) + _pull(acme) + + def fake(*args: object, **kwargs: object) -> _runner.Listing: + return _runner.Listing([_row("1")], True, ["1"], narrowed=True) + + monkeypatch.setattr(_runner, "read_listing", fake) + assert _pull(acme)["closed"] == 0 + _set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3") + assert _pull(acme)["closed"] == 1 + assert _open(acme) == {"workable:acme:1", "workable:acme:2"} + + +def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")] + for target in targets: + slug = target.partition(":")[2] + store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse") + + def fake_fetch(self: ATSSource) -> object: + [(target, _)] = self.targets() + yield _item("greenhouse", target.partition(":")[2], "1") + self.state[f"ats:ok:{target}"] = self.options["run"] + + real = Store.upsert_role + + def upsert(self: Store, item: Item, sub: object = None) -> bool: + if item.source_id.startswith("greenhouse:b:"): + raise ValueError("a row the store refuses") + return real(self, item, sub) # type: ignore[arg-type] + + monkeypatch.setattr(ATSSource, "fetch", fake_fetch) + monkeypatch.setattr(Store, "upsert_role", upsert) + result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type] + + assert result["roles"] == 2 + assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"} + assert "ats:ok:greenhouse:b" not in store.state + since, reason = pull.failing_boards(store)["greenhouse:b"] + assert since and reason == "ValueError: a row the store refuses" + assert set(pull.failing_boards(store)) == {"greenhouse:b"} + + +def test_a_board_whose_rows_do_not_store_keeps_its_failing_record( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + since = _days_ago(2) + acme.state[f"{pull.FAILING}workable:acme"] = json.dumps( + {"since": since, "reason": "404 Not Found"}) + _answer(monkeypatch, ([_row("1")], True)) + + def refuse(self: Store, item: Item, sub: object = None) -> bool: + raise ValueError("no") + + monkeypatch.setattr(Store, "upsert_role", refuse) + _pull(acme) + assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no") + assert "ats:ok:workable:acme" not in acme.state + + +# ------------------------------------------------------- the settle speaks + +def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many( + store: Store) -> None: + sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big", + board="greenhouse") + store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone", + board="greenhouse") + for n in range(80): + store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub) + run = pull.claim_run(store) + store.state["ats:ok:greenhouse:big"] = run + store.state["ats:complete:greenhouse:big"] = "1" + store.state["ats:ids:greenhouse:big"] = json.dumps( + [f"greenhouse:big:{n}" for n in range(20)]) + store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps( + {"since": _days_ago(1), "reason": "404 Not Found"}) + + said: list[object] = [] + result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type] + context=RunContext(on_notice=said.append)) + assert (result["unanswered"], result["closed"]) == (1, 60) + [notice] = said + assert notice.code == "pull.settled" # type: ignore[attr-defined] + assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined] + assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined] + assert "closed 60 roles" in notice.text # type: ignore[attr-defined] + + +def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(i) for i in "12"], True)) + said: list[object] = [] + results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type] + for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type] + pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type] + context=RunContext(on_notice=said.append)) + assert said == [] + + +def test_a_finder_that_raises_releases_the_pull_claim( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + def broken(store: Store, subs: object) -> dict[str, object]: + raise RuntimeError("reach broke") + + monkeypatch.setattr(pull, "reaches", broken) + # The worker runs the chain's finder once the job is done; here at once. + context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme)) + with pytest.raises(RuntimeError, match="reach broke"): + cascade.run_pull(FakeConfig(), acme, context=context) + assert pull.running(acme) == "" + + # And the job itself lets go when the chain cannot be built. + def no_chain(run: str) -> object: + raise RuntimeError("no chain") + + monkeypatch.setattr(cascade, "pull_chain", no_chain) + with pytest.raises(RuntimeError, match="no chain"): + cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True)) + assert pull.running(acme) == "" diff --git a/tests/test_search.py b/tests/test_search.py index dc34a6a..0cf9d9e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -371,3 +371,56 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed( assert store.roles() == [] # And a batch that starts after the switch reads nothing. assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type] + + +# ------------------------------------------------- failures and the guard + +def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None: + """The provider `fake` of `_fake_provider`, answering with an error.""" + _fake_provider(monkeypatch, [], complete=True) + + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + raise ValueError("hits renamed") + + readers.PROVIDERS["fake"].page = page + + +def test_a_search_whose_provider_raises_is_recorded_failing( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", + params={"q": "backend"}) + _fake_provider(monkeypatch, ["h1"], complete=True) + _pull(store) + _broken_provider(monkeypatch) + result = _pull(store) + assert result["unanswered"] == 1 + since, reason = pull.failing_boards(store)["fake:cph"] + assert since and reason == "ValueError: hits renamed" + assert {r.source_id for r in store.roles()} == {"fake:h1"} + + # Failing for `STALE_DAYS` closes its roles, as a board's. + old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat( + timespec="seconds") + store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason}) + _pull(store) + assert store.roles() == [] + + # Its next answer removes the record and reopens what it lists. + _fake_provider(monkeypatch, ["h1"], complete=True) + _pull(store) + assert pull.failing_boards(store) == {} + assert {r.source_id for r in store.roles()} == {"fake:h1"} + + +def test_a_search_that_lists_nothing_after_forty_is_suspect( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", + params={"q": "backend"}) + forty = [f"h{n}" for n in range(40)] + _fake_provider(monkeypatch, forty, complete=True) + _pull(store) + _fake_provider(monkeypatch, [], complete=True) + result = _pull(store) + assert (result["unanswered"], result["closed"]) == (1, 0) + assert len(store.roles()) == 40 + assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]