Guard each target in a batch, close remote-unasked and narrowed roles by age, settle failing searches, and raise one notice per settle naming what needs a look

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:53:47 +02:00
1 parent 24a165818c
commit 9c8a610787
4 files changed
+531 -70

No files matched your search

+10 -2
View File
@@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
Under `NULL` there is no queue to take the chain, so this raises rather Under `NULL` there is no queue to take the chain, so this raises rather
than report a pull that never happened; `app job pull` claims the run and than report a pull that never happened; `app job pull` claims the run and
starts the chain on a queue of its own instead of calling this. starts the chain on a queue of its own instead of calling this. Anything
that raises once the run is claimed, building the chain or handing it
over, releases the claim first, or the next pull would wait
`pull.ABANDONED` for a run that never started.
""" """
run = pull.claim_run(store) run = pull.claim_run(store)
if not run: if not run:
since = pull.running(store) since = pull.running(store)
context.notice(already_running(since)) context.notice(already_running(since))
return {"skipped": pull.ALREADY_RUNNING, "running": since} return {"skipped": pull.ALREADY_RUNNING, "running": since}
if not context.task(pull_chain(run)): try:
started = context.task(pull_chain(run))
except Exception:
pull.release_run(store, run)
raise
if not started:
pull.release_run(store, run) pull.release_run(store, run)
raise RuntimeError("no queue to start the pull cascade on") raise RuntimeError("no queue to start the pull cascade on")
return {"started": " > ".join(CASCADES["pull"]), "run": run} return {"started": " > ".join(CASCADES["pull"]), "run": run}
+237 -62
View File
@@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see
`plugins/readers/_runner.py`). `plugins/readers/_runner.py`).
A board read with a filter (`ats:filtered:<target>`, from the agents' reach, A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a `app.jobs.reach`, or its subscription's `params`) lists only the roles in
person acted on: those with a vote or an application (`_close_unlisted`). A reach, so its close keeps open the roles a person acted on: those with a vote
filtered read, or one of a reader that declares `empty="suspect"`, that lists or an application (`_close_unlisted`). When the reach wanted remote roles and
under a tenth of what the same read listed last time is recorded by the the reader could not ask for them (`ats:remote_unasked:<target>`), the read
runner as a failure, not as a list to close against. says nothing of the remote roles either, so they stay open too and close only
once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
that lists under a tenth of what the same read listed last time is recorded
by the runner as a failure, not as a list to close against (`_runner.read`).
A board swept across pulls (`ats:swept:<target>`, a complete read that finished A board swept across pulls (`ats:swept:<target>`, a complete read that finished
a sweep an earlier pull began) closes a role only when two finished sweeps in a a sweep an earlier pull began) closes a role only when two finished sweeps in a
@@ -33,14 +36,21 @@ not swept clears them.
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
the roles matching a word, which says nothing of the roles it did not list, the roles matching a word, which says nothing of the roles it did not list,
so it closes nothing; only an unfiltered or a place-filtered complete read so it closes by list nothing: a role of such a board closes once the board
closes. has not listed it for `STALE_DAYS`, the rule searches have.
A search closes the same way when it listed everything that matches (see A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its `plugins/search.py` for how each provider knows). One that did not closes its
roles only once no search has seen them for `STALE_DAYS`. A board roles only once no search has seen them for `STALE_DAYS`. A board or a
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`) search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
closes all of its roles, and its next answer reopens what it lists. `search:failing:<target>`) closes all of its roles, and its next answer
reopens what it lists. A target that raises in its batch after its read, a
row that does not store, is recorded as failing the same way and the batch
goes on with the next one (`run_batch`).
The settle says what needs a look in one notice (`settled`): the targets
that did not answer or were read incomplete, and the boards that closed more
than `CLOSE_NOTICE` roles or more than half of theirs.
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run` One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
set only while the previous run is settled, in one statement), so the set only while the previous run is settled, in one statement), so the
@@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta
from functools import partial from functools import partial
from typing import Any from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
from app.config import Config from app.config import Config
from app.jobs.reach import reaches from app.jobs.reach import reaches
@@ -91,6 +101,19 @@ STALE_DAYS = 21
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer. #: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
FAILING = "ats:failing:" FAILING = "ats:failing:"
#: The same for a search, kept by the `search` plugin.
SEARCH_FAILING = "search:failing:"
#: A board that closes more roles than this in one settle is named in the
#: settle's notice, and so is one that closes more than half of its open
#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
#: one is an ordinary day.
CLOSE_NOTICE = 50
CLOSE_SHARE_MIN = 10
#: How many targets a notice names in its text; its detail names them all.
NOTICE_NAMES = 10
#: The `state` key that records the starter searches were imported. #: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded" SEARCHES_SEEDED = "searches:seeded"
@@ -142,9 +165,14 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
`close_boards` compares against. `run` is a marker the caller claimed `close_boards` compares against. `run` is a marker the caller claimed
already (the `pull` job, `app job pull`); without one the finder claims already (the `pull` job, `app job pull`); without one the finder claims
its own, and queues nothing when another run is in flight. A run with its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it.""" nothing to read is settled here, since no settle will see a batch of it,
and so is a run whose finder raises: no batch and no settle would follow,
and the next pull would wait `ABANDONED` for a claim nobody holds."""
held = run
try:
seed_searches(config, store) seed_searches(config, store)
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES} subs = {source: store.subscriptions(source=source, fetchable=True)
for source in SOURCES}
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES] targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
if not any(found for _, found in targets): if not any(found for _, found in targets):
if run: if run:
@@ -155,14 +183,20 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING, log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store)) running(store))
return [] return []
held = run
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`). # The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
reach_of = reaches(store, subs["ats"]) reach_of = reaches(store, subs["ats"])
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run} batches: list[dict[str, Any]] = [
{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)] for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
for batch in batches: for batch in batches:
if batch["source"] == "ats": if batch["source"] == "ats":
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of} batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
return batches return batches
except Exception:
if held:
release_run(store, held)
raise
@job("pull_batch", "Reading boards", resource="http") @job("pull_batch", "Reading boards", resource="http")
@@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
plugin as its `run` option, so its stamps belong to that run whatever the plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs. `reach` is `target ->` the stored marker says by the time the batch runs. `reach` is `target ->` the
reach the run computed for a board (`app.jobs.reach`), handed to the reach the run computed for a board (`app.jobs.reach`), handed to the
plugin as its `reach` option. plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
which the runner's 10% guard accepts the answer it kept refusing.
Each target is guarded on its own. A read that fails is the plugin's to
record; anything raised after it, a row `upsert_role` refuses or a bug
past the read, is recorded here under `<source>:failing:<target>` with
its reason, and the batch goes on with the next target. Otherwise one
bad row would cost the targets after it in the batch their read on every
pull, since batches are stable, and leave no trace on any of them. A
Kill (`Cancelled`) still ends the batch.
""" """
from plugins.ats import ATSSource from plugins.ats import ATSSource
from plugins.readers import _runner
from plugins.search import SearchSource from plugins.search import SearchSource
# Fetchable now, not when the pull was queued: a board switched off on the # Fetchable now, not when the pull was queued: a board switched off on the
@@ -200,10 +244,14 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
if sub is None: if sub is None:
continue continue
context.check() context.check()
options = dict(config.sources.get(source) or {}) | {"run": run} options = dict(config.sources.get(source) or {}) | {"run": run,
"stale_days": STALE_DAYS}
items: Any = None
try:
plugin: ATSSource | SearchSource plugin: ATSSource | SearchSource
if source == "search": if source == "search":
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}] options["searches"] = [{"url": target, "label": sub.label,
"params": sub.params}]
plugin = SearchSource(options) plugin = SearchSource(options)
else: else:
# A board's params are its filter; a reader's key is kept with # A board's params are its filter; a reader's key is kept with
@@ -214,10 +262,21 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
options["reach"] = (reach or {}).get(target) options["reach"] = (reach or {}).get(target)
plugin = ATSSource(options) plugin = ATSSource(options)
plugin.state = store.state plugin.state = store.state
for item in plugin.fetch(): items = plugin.fetch()
for item in items:
item.source = source item.source = source
store.upsert_role(item, sub) store.upsert_role(item, sub)
stored += 1 stored += 1
except Cancelled:
raise
except Exception as exc:
log.warning("pull: %s failed after its read (%s); on to the next target",
target, exc, exc_info=True)
_runner.record_failure(store.state, f"{source}:failing:{target}", exc)
finally:
# The plugin's generator holds its HTTP client until it is closed.
if items is not None and hasattr(items, "close"):
items.close()
return {"roles": stored, "run": run} return {"roles": stored, "run": run}
@@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str:
def close_boards(config: Config, store: Store, tally: Tally, *, def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]: context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and """The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened. stopped listing a role from, tally what happened, and say what needs a
look.
Reads `self.state` rather than the batch jobs' own results, because a Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it `pull_batch` job that raised partway through still leaves the boards it
@@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`ats:ok:<target>` was not written during *this* run — it failed, or its `ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed by its list, and a board whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only that answered incompletely is not either. A search closes by its list only
when that list was complete, and otherwise by `STALE_DAYS`. A board that when that list was complete, and otherwise by `STALE_DAYS`. A board or a
has failed on every fetch since `STALE_DAYS` ago closes all its roles. search that has failed on every fetch since `STALE_DAYS` ago closes all
its roles (`failing_boards`). How a read that answered closes is
`_close_read`'s.
The stamps are compared with this run's own marker, the one its batches The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have carried, never with the stored `pull:run`, which a later run may have
replaced. A settle with no batch behind it (a refused pull, nothing to replaced. A settle with no batch behind it (a refused pull, nothing to
read) closes nothing by list or by age. Last, the run is marked settled, read) closes nothing by list or by age and says nothing. Last, the run is
which lets the next pull start. marked settled, which lets the next pull start.
A board or search switched off while its batch was in flight had its A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here. switched-off one is closed again here; those closes are asked for and not
named in the notice.
The counts go into the result (`unanswered` counts boards and searches
whose stamp is not this run's), and one notice (`settled`) names every
target unanswered, every board read incomplete other than one mid-sweep,
and every board that closed more than `CLOSE_NOTICE` roles or more than
half the open roles it had, when there is any. The result alone was
seen by nobody: a settle closed 12,588 roles with nothing on screen.
The `prep` stage that follows maps what this run stored and folds the The `prep` stage that follows maps what this run stored and folds the
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
and the roles this run added join theirs. and the roles this run added join theirs.
""" """
answered = searched = failed = incomplete = closed = 0 answered = searched = incomplete = closed = 0
unanswered: list[str] = []
partial: list[str] = []
many: list[tuple[str, int, int]] = []
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds") stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store) failing = failing_boards(store)
run_marker = _run_of(tally, context) run_marker = _run_of(tally, context)
open_before = _open_counts(store) if run_marker else {}
for source in SOURCES: for source in SOURCES:
for sub in store.subscriptions(source=source): for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch): if not (sub.subscribed and sub.fetch):
@@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
for sub in store.subscriptions(source=source, fetchable=True): for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "") stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker): if not (stamp and stamp == run_marker):
failed += 1 unanswered.append(sub.target)
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else "" since = failing.get(sub.target, ("", ""))[0]
if since and since < stale: shut = store.close_subscription(sub.id) if since and since < stale else 0
closed += store.close_subscription(sub.id) else:
continue
if source == "search": if source == "search":
searched += 1 searched += 1
else: else:
answered += 1 answered += 1
if not store.state.get(f"{source}:complete:{sub.target}", ""): read = _close_read(store, source, sub, stale)
if read is None:
if source == "ats": if source == "ats":
incomplete += 1 incomplete += 1
if not store.state.get(f"ats:sweep:{sub.target}"):
partial.append(sub.target)
continue continue
if store.state.get(f"{source}:narrowed:{sub.target}", ""): shut = read
continue closed += shut
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) before = open_before.get(sub.id, 0)
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", "")) if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
if store.state.get(f"{source}:swept:{sub.target}", ""): many.append((sub.target, shut, before))
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
continue
store.state.pop(f"{source}:missing:{sub.target}", None)
if is_filtered:
closed += _close_unlisted(store, source, seen_ids, sub.id)
else:
closed += store.mark_closed(source, seen_ids, sub.id)
if searched: if searched:
closed += store.close_unseen("search", stale) closed += store.close_unseen("search", stale)
if run_marker: if run_marker:
if unanswered or partial or many:
context.notice(settled(unanswered, partial, many, failing,
boards=answered, searches=searched, closed=closed))
release_run(store, run_marker) release_run(store, run_marker)
roles = sum(int(result.get("roles", 0)) for result in tally.results) roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed, return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
"incomplete": incomplete, "roles": roles, "closed": closed} "incomplete": incomplete, "roles": roles, "closed": closed}
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int: def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
"""Close what one target's read of this run says has closed, and return
how many; None when the read was incomplete, which closes nothing.
A read narrowed by terms closes only the roles it has not listed for
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
two sweeps in a row missed (`_close_swept`); any other complete read
clears that record and closes what it did not list, all of it unfiltered
(`Store.mark_closed`), filtered sparing the roles acted on
(`_close_unlisted`). A filtered read that could not ask for remote roles
(`remote_unasked`) spares the remote ones as well, and closes those by
age instead."""
state, target = store.state, sub.target
if not state.get(f"{source}:complete:{target}", ""):
return None
seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
if state.get(f"{source}:narrowed:{target}", ""):
return _close_aged(store, source, sub.id, seen_ids, stale)
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
if spare_remote else 0)
if state.get(f"{source}:swept:{target}", ""):
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
spare_remote=spare_remote)
state.pop(f"{source}:missing:{target}", None)
if is_filtered:
return aged + _close_unlisted(store, source, seen_ids, sub.id,
spare_remote=spare_remote)
return store.mark_closed(source, seen_ids, sub.id)
def _open_counts(store: Store) -> dict[int, int]:
"""`subscription id -> open roles`, before the settle closes any, for the
share a board closed."""
with store.transaction() as conn:
return {int(row[0]): int(row[1]) for row in conn.execute(
"""SELECT subscription_id, count(*) FROM roles
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
GROUP BY subscription_id""")}
def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
closed: int) -> Notice:
"""The notice of a settle that has something to look at: the targets
that did not answer (with the reason their failing record gives), the
boards read incomplete, and the boards that closed many roles, each
as `closed of open`, with the totals. The text names the first
`NOTICE_NAMES` of each, the detail all of them, one a line."""
def named(target: str) -> str:
reason = failing.get(target, ("", ""))[1]
return f"{target} ({reason})" if reason else target
groups = [("Not answering", [named(t) for t in unanswered]),
("Read incomplete", list(incomplete)),
("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
text = [f"The pull read {boards} boards and {searches} searches and closed "
f"{closed:,} roles."]
detail: list[str] = []
for title, names in groups:
if not names:
continue
shown = ", ".join(names[:NOTICE_NAMES])
more = len(names) - NOTICE_NAMES
text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
else "."))
detail += [f"{title}: {name}" for name in names]
return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
#: The roles a partial read never closes by its list: one with a vote (any,
#: neutral included) or an application (which holds the notes).
_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)"""
#: The remote roles a filtered read that could not ask for them leaves alone.
_NOT_REMOTE = " AND remote != 'remote'"
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
*, spare_remote: bool = False) -> int:
"""`Store.mark_closed` for a filtered read: close every open role of the """`Store.mark_closed` for a filtered read: close every open role of the
board it did not list, except a role someone acted on, one with a vote board it did not list, except a role someone acted on (`_ACTED_ON`). A
(any, neutral included) or an application (which holds the notes). A
filtered read cannot tell a role gone from a role outside its filter, and filtered read cannot tell a role gone from a role outside its filter, and
a role acted on stays until the board is read without the filter or a role acted on stays until the board is read without the filter or
fails for `STALE_DAYS`.""" fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
well, since the read could not ask for them."""
with store.transaction() as conn: with store.transaction() as conn:
return conn.execute( return conn.execute(
"""UPDATE roles SET closed_at = ? f"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?)) AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
AND uid NOT IN (SELECT uid FROM role_votes) + (_NOT_REMOTE if spare_remote else ""),
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)""",
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount (store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
stale: str, *, remote_only: bool = False) -> int:
"""Close the open roles of one board that its read did not list and that
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
writes on every listing): the rule for what a read cannot speak for. A
read narrowed by terms closes all such roles, as a search does; with
`remote_only`, a filtered read that could not ask for remote closes only
its remote roles so, sparing those acted on as its list does."""
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
with store.transaction() as conn:
return conn.execute(
f"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND last_seen < ?
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
(store.now(), source, subscription_id, stale,
json.dumps(list(seen_ids)))).rowcount
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str], def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
filtered: bool) -> int: filtered: bool, *, spare_remote: bool = False) -> int:
"""The close of a board whose complete read finished a sweep across """The close of a board whose complete read finished a sweep across
pulls: its open roles missing from `seen_ids` that also are in pulls: its open roles missing from `seen_ids` that also are in
`<source>:missing:<target>`, the ones the previous finished sweep missed. `<source>:missing:<target>`, the ones the previous finished sweep missed.
@@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
missed may only have shifted back past the cursor when roles closed missed may only have shifted back past the cursor when roles closed
between pulls, so one sweep is not evidence that it is gone. A filtered between pulls, so one sweep is not evidence that it is gone. A filtered
read keeps its exceptions as `_close_unlisted` does: a role acted on read keeps its exceptions as `_close_unlisted` does: a role acted on
stays open, and stays in `missing`.""" stays open, and stays in `missing`, and so does a remote role with
`spare_remote`."""
key = f"{source}:missing:{sub.target}" key = f"{source}:missing:{sub.target}"
try: try:
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")} earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
except (TypeError, ValueError): except (TypeError, ValueError):
earlier = set() earlier = set()
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes) acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)""" if filtered else ""
with store.transaction() as conn: with store.transaction() as conn:
missed = [row[0] for row in conn.execute( missed = [row[0] for row in conn.execute(
"""SELECT source_id FROM roles """SELECT source_id FROM roles
@@ -365,15 +536,19 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
def failing_boards(store: Store) -> dict[str, tuple[str, str]]: def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
"""`target -> (since, reason)` for every `ats` board whose last fetch """`target -> (since, reason)` for every `ats` board and every search
failed: when the failures began, and what the latest one said.""" whose last fetch failed: when the failures began, and what the latest
one said. No board host is also a search provider, so a board's target
(`host:slug`) and a search's (`provider:name`) never collide in one
dict."""
out: dict[str, tuple[str, str]] = {} out: dict[str, tuple[str, str]] = {}
for key, value in store.state_items(FAILING).items(): for prefix in (FAILING, SEARCH_FAILING):
for key, value in store.state_items(prefix).items():
try: try:
record = json.loads(value) record = json.loads(value)
except ValueError: except ValueError:
continue continue
out[key[len(FAILING):]] = (str(record.get("since") or ""), out[key[len(prefix):]] = (str(record.get("since") or ""),
str(record.get("reason") or "")) str(record.get("reason") or ""))
return out return out
+231 -6
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import json import json
import sys import sys
import threading import threading
import types
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from pathlib import Path from pathlib import Path
@@ -21,8 +22,8 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402 from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext, Tally # noqa: E402 from pipeline import Cancelled, RunContext, Tally # noqa: E402
from app.models import Application # noqa: E402 from app.models import Application # noqa: E402
from plugins import readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import _runner # noqa: E402 from plugins.readers import Filters, Reads, _runner # noqa: E402
from plugins.ats import ATSSource # noqa: E402 from plugins.ats import ATSSource # noqa: E402
from app import prep # noqa: E402 from app import prep # noqa: E402
from app.jobs import cascade, pull # noqa: E402 from app.jobs import cascade, pull # noqa: E402
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0, assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0,
"roles": 1, "closed": 1} "roles": 1, "closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -163,7 +164,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0, assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0,
"roles": 0, "closed": 0} "roles": 0, "closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
@@ -256,7 +257,7 @@ def test_a_failing_board_keeps_its_roles_before_stale_days(
_answer(monkeypatch, ValueError("gone")) _answer(monkeypatch, ValueError("gone"))
result = _pull(acme) result = _pull(acme)
assert _open(acme) == {"workable:acme:1"} assert _open(acme) == {"workable:acme:1"}
assert (result["failed"], result["closed"]) == (1, 0) assert (result["unanswered"], result["closed"]) == (1, 0)
since, reason = pull.failing_boards(acme)["workable:acme"] since, reason = pull.failing_boards(acme)["workable:acme"]
assert since and reason == "ValueError: gone" assert since and reason == "ValueError: gone"
@@ -486,7 +487,7 @@ def test_run_a_settle_does_not_close_on_run_b_ids(
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type] b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
assert (result["failed"], result["closed"]) == (1, 0) assert (result["unanswered"], result["closed"]) == (1, 0)
assert _open(acme) == {"workable:acme:1", "workable:acme:2"} assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
assert pull.running(acme) == b["run"] assert pull.running(acme) == b["run"]
@@ -507,3 +508,227 @@ def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> No
context = RunContext(waited_for=(failed,)) context = RunContext(waited_for=(failed,))
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
assert result["boards"] == 1 and pull.running(acme) == "" assert result["boards"] == 1 and pull.running(acme) == ""
# -------------------------------------------------------- losses, guarded
def _days_ago(days: float) -> str:
return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds")
def _set(store: Store, column: str, value: str, *source_ids: str) -> None:
with store.transaction() as conn:
conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?",
[(value, i) for i in source_ids])
@pytest.fixture
def big(store: Store) -> Store:
store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
board="greenhouse")
return store
def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails(
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
_pull(big)
_answer(monkeypatch, ([], True))
result = _pull(big)
assert len(_open(big)) == 40
assert (result["unanswered"], result["closed"]) == (1, 0)
assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1]
def test_a_whole_read_of_nothing_after_five_closes_the_board(
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(str(n)) for n in range(5)], True))
_pull(big)
_answer(monkeypatch, ([], True))
assert _pull(big)["closed"] == 5
assert _open(big) == set()
def test_a_suspect_answer_is_accepted_after_stale_days(
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
_pull(big)
_answer(monkeypatch, ([_row("1")], True))
_pull(big)
assert len(_open(big)) == 40
record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"])
big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps(
record | {"since": _days_ago(pull.STALE_DAYS + 1)})
result = _pull(big)
assert _open(big) == {"greenhouse:big:1"}
assert (result["unanswered"], result["closed"]) == (0, 39)
assert pull.failing_boards(big) == {}
assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1
def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]:
"""A whole-listing host `fake` that filters by country and, per `remote`,
on remote; its feed lists the ids in the returned list."""
listed: list[str] = []
module: object = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined]
module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined]
module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined]
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
module.READS = Reads( # type: ignore[attr-defined]
scope="board", listing="whole", text="listing", complete="whole", places="one",
robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type]
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined]
monkeypatch.setitem(readers.HOSTS, "fake", module)
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps(
listed).encode())
return listed
def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
listed = _remote_host(monkeypatch, remote="none")
store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake")
reach = {"fake:acme": {"countries": ["DK"], "remote": True}}
def pull_once() -> dict[str, object]:
run = pull.claim_run(store)
result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type]
reach=reach)
return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
listed[:] = ["1", "2", "3", "4"]
pull_once()
_set(store, "remote", "remote", "fake:acme:3", "fake:acme:4")
listed[:] = ["1"]
assert pull_once()["closed"] == 1
assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"}
assert store.state["ats:remote_unasked:fake:acme"] == "1"
_set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4")
assert pull_once()["closed"] == 1
assert _open(store) == {"fake:acme:1", "fake:acme:3"}
def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(i) for i in "123"], True))
_pull(acme)
def fake(*args: object, **kwargs: object) -> _runner.Listing:
return _runner.Listing([_row("1")], True, ["1"], narrowed=True)
monkeypatch.setattr(_runner, "read_listing", fake)
assert _pull(acme)["closed"] == 0
_set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3")
assert _pull(acme)["closed"] == 1
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")]
for target in targets:
slug = target.partition(":")[2]
store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse")
def fake_fetch(self: ATSSource) -> object:
[(target, _)] = self.targets()
yield _item("greenhouse", target.partition(":")[2], "1")
self.state[f"ats:ok:{target}"] = self.options["run"]
real = Store.upsert_role
def upsert(self: Store, item: Item, sub: object = None) -> bool:
if item.source_id.startswith("greenhouse:b:"):
raise ValueError("a row the store refuses")
return real(self, item, sub) # type: ignore[arg-type]
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
monkeypatch.setattr(Store, "upsert_role", upsert)
result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type]
assert result["roles"] == 2
assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"}
assert "ats:ok:greenhouse:b" not in store.state
since, reason = pull.failing_boards(store)["greenhouse:b"]
assert since and reason == "ValueError: a row the store refuses"
assert set(pull.failing_boards(store)) == {"greenhouse:b"}
def test_a_board_whose_rows_do_not_store_keeps_its_failing_record(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
since = _days_ago(2)
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
{"since": since, "reason": "404 Not Found"})
_answer(monkeypatch, ([_row("1")], True))
def refuse(self: Store, item: Item, sub: object = None) -> bool:
raise ValueError("no")
monkeypatch.setattr(Store, "upsert_role", refuse)
_pull(acme)
assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no")
assert "ats:ok:workable:acme" not in acme.state
# ------------------------------------------------------- the settle speaks
def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many(
store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
board="greenhouse")
store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone",
board="greenhouse")
for n in range(80):
store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub)
run = pull.claim_run(store)
store.state["ats:ok:greenhouse:big"] = run
store.state["ats:complete:greenhouse:big"] = "1"
store.state["ats:ids:greenhouse:big"] = json.dumps(
[f"greenhouse:big:{n}" for n in range(20)])
store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps(
{"since": _days_ago(1), "reason": "404 Not Found"})
said: list[object] = []
result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type]
context=RunContext(on_notice=said.append))
assert (result["unanswered"], result["closed"]) == (1, 60)
[notice] = said
assert notice.code == "pull.settled" # type: ignore[attr-defined]
assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined]
assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined]
assert "closed 60 roles" in notice.text # type: ignore[attr-defined]
def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(i) for i in "12"], True))
said: list[object] = []
results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type]
pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type]
context=RunContext(on_notice=said.append))
assert said == []
def test_a_finder_that_raises_releases_the_pull_claim(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
def broken(store: Store, subs: object) -> dict[str, object]:
raise RuntimeError("reach broke")
monkeypatch.setattr(pull, "reaches", broken)
# The worker runs the chain's finder once the job is done; here at once.
context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme))
with pytest.raises(RuntimeError, match="reach broke"):
cascade.run_pull(FakeConfig(), acme, context=context)
assert pull.running(acme) == ""
# And the job itself lets go when the chain cannot be built.
def no_chain(run: str) -> object:
raise RuntimeError("no chain")
monkeypatch.setattr(cascade, "pull_chain", no_chain)
with pytest.raises(RuntimeError, match="no chain"):
cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True))
assert pull.running(acme) == ""
+53
View File
@@ -371,3 +371,56 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
assert store.roles() == [] assert store.roles() == []
# And a batch that starts after the switch reads nothing. # And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type] assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
# ------------------------------------------------- failures and the guard
def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None:
"""The provider `fake` of `_fake_provider`, answering with an error."""
_fake_provider(monkeypatch, [], complete=True)
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
raise ValueError("hits renamed")
readers.PROVIDERS["fake"].page = page
def test_a_search_whose_provider_raises_is_recorded_failing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_fake_provider(monkeypatch, ["h1"], complete=True)
_pull(store)
_broken_provider(monkeypatch)
result = _pull(store)
assert result["unanswered"] == 1
since, reason = pull.failing_boards(store)["fake:cph"]
assert since and reason == "ValueError: hits renamed"
assert {r.source_id for r in store.roles()} == {"fake:h1"}
# Failing for `STALE_DAYS` closes its roles, as a board's.
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
timespec="seconds")
store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason})
_pull(store)
assert store.roles() == []
# Its next answer removes the record and reopens what it lists.
_fake_provider(monkeypatch, ["h1"], complete=True)
_pull(store)
assert pull.failing_boards(store) == {}
assert {r.source_id for r in store.roles()} == {"fake:h1"}
def test_a_search_that_lists_nothing_after_forty_is_suspect(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
forty = [f"h{n}" for n in range(40)]
_fake_provider(monkeypatch, forty, complete=True)
_pull(store)
_fake_provider(monkeypatch, [], complete=True)
result = _pull(store)
assert (result["unanswered"], result["closed"]) == (1, 0)
assert len(store.roles()) == 40
assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]