Guard every read that is not a sweep, treat a read that asks nothing as incomplete, and mark a filtered read that could not ask remote, so no board loses its roles to a changed answer or a filter it could not send

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:53:47 +02:00
1 parent 4e827665dd
commit 36bc639384
4 files changed
+332 -116

No files matched your search

+41 -23
View File
@@ -16,7 +16,9 @@ The allow-list and the classifier are ported from `openings.go` and
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is read independently; one that fails is logged,
recorded as failing under `ats:failing:<target>`, and skipped. A board's
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
whose read lists under a tenth of what it listed last time (`_runner.read`'s
guard), until it has failed that way for the pull's `stale_days`. A board's
filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`) win, narrowed to what its reader declares
@@ -26,7 +28,8 @@ the whole board: it prunes no per-role state.
A board's size (`board_size`) is set once, when the board is added
(`app.jobs.sources.set_board`) or rechecked, and kept under
`ats:size:<target>` with the day it was checked under
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small
`ats:sizechecked:<target>` (`keep_size`, which also resets the guard's count,
so a Recheck accepts a board that shrank); the reach reads it to leave small
boards whole. A pull does not rewrite it. A board with no size yet is sized
in its batch before it is read: its reach then says `unsized` and the size
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
@@ -106,8 +109,9 @@ _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
def _limits() -> _runner.Limits:
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
def _limits(stale_days: int | None = None) -> _runner.Limits:
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP,
stale_days=stale_days)
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
@@ -339,8 +343,7 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
if (_runner.declared(module) and module.READS.listing == "paged"
and module.READS.complete == "stated"):
if module.READS.listing == "paged" and module.READS.complete == "stated":
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
if isinstance(total, int):
return total
@@ -348,11 +351,18 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
def keep_size(state: Any, target: str, size: int, *, today: str = "",
reset: bool = True) -> None:
"""Keep board `target`'s size and the day it was checked (ISO,
`today`, else today in UTC)."""
`today`, else today in UTC). With `reset`, the default and what a
Recheck does, the 10% guard's count goes too (`ats:count:<target>`), so
the next read of a board that really shrank is taken as it is instead of
failing as suspect; the plugin sizing a board as it reads it passes
False, since its count is the read it just stamped."""
state[f"ats:size:{target}"] = str(size)
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
if reset:
state.pop(f"ats:count:{target}", None)
failure_reason = _runner.failure_reason
@@ -400,14 +410,15 @@ class ATSSource(Source):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
ask, unsized = self._ask(client, target, module)
ask, unsized, remote = self._ask(client, target, module)
stale_days = self.options.get("stale_days")
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
limits=_limits())
limits=_limits(int(stale_days) if stale_days else None),
remote=remote)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
_runner.record_failure(self.state, failing, exc)
return
self.state.pop(failing, None)
def compound(source_id: str) -> str:
# Matches `roles.source_id`, which `Store.mark_closed` compares.
@@ -435,11 +446,13 @@ class ATSSource(Source):
yield item_of(row, target, label)
# Stamped only once every row went out, so a board that blew up half
# way through is neither marked ok nor has a partial id list treated
# as the whole of it.
# as the whole of it, and it stays failing until then: a row that
# fails to store (`app.jobs.pull.run_batch`) leaves the record in place.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
self.state.pop(failing, None)
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
keep_size(self.state, target, len(listing.ids))
keep_size(self.state, target, len(listing.ids), reset=False)
if detail_sets is not None:
# A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
@@ -447,9 +460,11 @@ class ATSSource(Source):
self.state[key] = json.dumps(sorted(ids))
def _ask(self, client: httpx.Client, target: str,
module: Any) -> tuple[readers.Ask, bool]:
module: Any) -> tuple[readers.Ask, bool, bool]:
"""What `target` is asked for, from the reach its batch handed it and
its params, and whether it still needs a size. A board with no size
its params, whether it still needs a size, and whether that reach
and params want remote roles (`_runner.remote_wanted`, which the
runner compares with what it could ask). A board with no size
yet (`unsized` in its reach) whose reach would narrow the read is
sized first (`board_size`, kept by `keep_size`), and read whole under
`small` roles. One whose read is whole anyway is sized by that read,
@@ -457,16 +472,18 @@ class ATSSource(Source):
given = self.options.get("reach")
params, keys = self.params_for(target), self.options.get("keys")
if not isinstance(given, dict) or not given.get("unsized"):
return _runner.ask_for(module, params, given, keys=keys), False
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
ask = _runner.ask_for(module, params, reach or None, keys=keys)
return (_runner.ask_for(module, params, given, keys=keys), False,
_runner.remote_wanted(module, params, given))
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None
ask = _runner.ask_for(module, params, reach, keys=keys)
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
return ask, True
return ask, True, _runner.remote_wanted(module, params, reach)
size = board_size(target, client=client)
keep_size(self.state, target, size)
keep_size(self.state, target, size, reset=False)
if size < int(given.get("small") or 0):
reach = None
ask = _runner.ask_for(module, params, None, keys=keys)
return ask, False
return ask, False, _runner.remote_wanted(module, params, reach)
# ------------------------------------------------------ the per-role reads
@@ -567,7 +584,8 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
def item_of(row: Row, target: str, label: str) -> Item:
"""One row of board `target` as the `Item` the store takes: its id
compound, `host:slug:id`, as `roles.source_id` has it."""
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
date is dated now, as `Item` itself would date it."""
host, _, slug = target.partition(":")
description = row.get("description") or ""
return Item(
@@ -576,7 +594,7 @@ def item_of(row: Row, target: str, label: str) -> Item:
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
published_at=row.get("posted") or datetime.now(UTC),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {