From 36bc639384add5c69301d1927bdfee94fff19ca1 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 15:53:47 +0200 Subject: [PATCH] Guard every read that is not a sweep, treat a read that asks nothing as incomplete, and mark a filtered read that could not ask remote, so no board loses its roles to a changed answer or a filter it could not send Co-Authored-By: Claude Fable 5.1 --- plugins/ats.py | 64 +++++++---- plugins/readers/_runner.py | 226 +++++++++++++++++++++++++------------ tests/test_readers.py | 134 +++++++++++++++++++++- tests/test_runner.py | 24 ++-- 4 files changed, 332 insertions(+), 116 deletions(-) diff --git a/plugins/ats.py b/plugins/ats.py index 0533775..08308a6 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -16,7 +16,9 @@ The allow-list and the classifier are ported from `openings.go` and A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what its slug is. Every board is read independently; one that fails is logged, -recorded as failing under `ats:failing:`, and skipped. A board's +recorded as failing under `ats:failing:`, and skipped, and so is one +whose read lists under a tenth of what it listed last time (`_runner.read`'s +guard), until it has failed that way for the pull's `stale_days`. A board's filter is the `reach` option the pull hands it (the agents', see `app/jobs/reach.py`), over which its subscription `params` (`{"countries": ["DK"], "remote": true}`) win, narrowed to what its reader declares @@ -26,7 +28,8 @@ the whole board: it prunes no per-role state. A board's size (`board_size`) is set once, when the board is added (`app.jobs.sources.set_board`) or rechecked, and kept under `ats:size:` with the day it was checked under -`ats:sizechecked:` (`keep_size`); the reach reads it to leave small +`ats:sizechecked:` (`keep_size`, which also resets the guard's count, +so a Recheck accepts a board that shrank); the reach reads it to leave small boards whole. A pull does not rewrite it. A board with no size yet is sized in its batch before it is read: its reach then says `unsized` and the size under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board @@ -106,8 +109,9 @@ _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")} -def _limits() -> _runner.Limits: - return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP) +def _limits(stale_days: int | None = None) -> _runner.Limits: + return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP, + stale_days=stale_days) def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: @@ -339,8 +343,7 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int: host, _, slug = target.partition(":") module = readers.HOSTS[host] get = _getter(client, host, slug) - if (_runner.declared(module) and module.READS.listing == "paged" - and module.READS.complete == "stated"): + if module.READS.listing == "paged" and module.READS.complete == "stated": _data, _results, total, _cursor = module.page(get, slug, None, readers.Ask()) if isinstance(total, int): return total @@ -348,11 +351,18 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int: return len(_runner.read_listing(module, slug, get, limits=limits).ids) -def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None: +def keep_size(state: Any, target: str, size: int, *, today: str = "", + reset: bool = True) -> None: """Keep board `target`'s size and the day it was checked (ISO, - `today`, else today in UTC).""" + `today`, else today in UTC). With `reset`, the default and what a + Recheck does, the 10% guard's count goes too (`ats:count:`), so + the next read of a board that really shrank is taken as it is instead of + failing as suspect; the plugin sizing a board as it reads it passes + False, since its count is the read it just stamped.""" state[f"ats:size:{target}"] = str(size) state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat() + if reset: + state.pop(f"ats:count:{target}", None) failure_reason = _runner.failure_reason @@ -400,14 +410,15 @@ class ATSSource(Source): raise ValueError(f"{target!r} is not host:slug for a known ATS host") module = readers.HOSTS[host] get = _getter(client, host, slug) - ask, unsized = self._ask(client, target, module) + ask, unsized, remote = self._ask(client, target, module) + stale_days = self.options.get("stale_days") listing = _runner.read(self.state, "ats", target, module, slug, get, ask, - limits=_limits()) + limits=_limits(int(stale_days) if stale_days else None), + remote=remote) except Exception as exc: log.warning("ats: %s failed (%s)", target, exc) _runner.record_failure(self.state, failing, exc) return - self.state.pop(failing, None) def compound(source_id: str) -> str: # Matches `roles.source_id`, which `Store.mark_closed` compares. @@ -435,11 +446,13 @@ class ATSSource(Source): yield item_of(row, target, label) # Stamped only once every row went out, so a board that blew up half # way through is neither marked ok nor has a partial id list treated - # as the whole of it. + # as the whole of it, and it stays failing until then: a row that + # fails to store (`app.jobs.pull.run_batch`) leaves the record in place. _runner.stamp(self.state, "ats", target, run_marker, listing, [compound(i) for i in listing.ids]) + self.state.pop(failing, None) if unsized and listing.complete and not (listing.filtered or listing.narrowed): - keep_size(self.state, target, len(listing.ids)) + keep_size(self.state, target, len(listing.ids), reset=False) if detail_sets is not None: # A complete listing drops the ids it no longer lists. for key, ids in zip(detail_keys(target), detail_sets, strict=False): @@ -447,9 +460,11 @@ class ATSSource(Source): self.state[key] = json.dumps(sorted(ids)) def _ask(self, client: httpx.Client, target: str, - module: Any) -> tuple[readers.Ask, bool]: + module: Any) -> tuple[readers.Ask, bool, bool]: """What `target` is asked for, from the reach its batch handed it and - its params, and whether it still needs a size. A board with no size + its params, whether it still needs a size, and whether that reach + and params want remote roles (`_runner.remote_wanted`, which the + runner compares with what it could ask). A board with no size yet (`unsized` in its reach) whose reach would narrow the read is sized first (`board_size`, kept by `keep_size`), and read whole under `small` roles. One whose read is whole anyway is sized by that read, @@ -457,16 +472,18 @@ class ATSSource(Source): given = self.options.get("reach") params, keys = self.params_for(target), self.options.get("keys") if not isinstance(given, dict) or not given.get("unsized"): - return _runner.ask_for(module, params, given, keys=keys), False - reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} - ask = _runner.ask_for(module, params, reach or None, keys=keys) + return (_runner.ask_for(module, params, given, keys=keys), False, + _runner.remote_wanted(module, params, given)) + reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None + ask = _runner.ask_for(module, params, reach, keys=keys) if not (_runner.filtered(ask) or _runner.narrowed(module, ask)): - return ask, True + return ask, True, _runner.remote_wanted(module, params, reach) size = board_size(target, client=client) - keep_size(self.state, target, size) + keep_size(self.state, target, size, reset=False) if size < int(given.get("small") or 0): + reach = None ask = _runner.ask_for(module, params, None, keys=keys) - return ask, False + return ask, False, _runner.remote_wanted(module, params, reach) # ------------------------------------------------------ the per-role reads @@ -567,7 +584,8 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http def item_of(row: Row, target: str, label: str) -> Item: """One row of board `target` as the `Item` the store takes: its id - compound, `host:slug:id`, as `roles.source_id` has it.""" + compound, `host:slug:id`, as `roles.source_id` has it. A row without a + date is dated now, as `Item` itself would date it.""" host, _, slug = target.partition(":") description = row.get("description") or "" return Item( @@ -576,7 +594,7 @@ def item_of(row: Row, target: str, label: str) -> Item: title=row["title"], excerpt=description[:300], body=description, - published_at=row.get("posted"), + published_at=row.get("posted") or datetime.now(UTC), subscription=label, kind="role", raw=_runner.raw_of(row) | { diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index ebf070c..edd6915 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -5,23 +5,25 @@ A reader says how its host answers; this module reads it: the guarded fetch (`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves to (`ask_for`) and its series (`series`), the listing whole or paged, one series per `one`-filter value unioned by id (`read_listing`), completeness, -the sweep of an oversized paged board across pulls, the 10% guard on a -filtered or `empty="suspect"` read (`read`), the per-role detail reads +the sweep of an oversized paged board across pulls, the 10% guard on every +read that is not a sweep (`read`), the per-role detail reads (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and `plugins/search.py` are thin `Source` classes over it: they pass their own limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay -as they were. - -A module without `READS` is read with the older signatures, `feed_url(slug)` -and `page(get, slug, cursor)`, and never gets an `Ask`. +as they were. Every reader declares `READS`; a module without it is refused +by the package before it gets here. State keys, under the Source's prefix and the target (`ats:sweep:`): sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor", "seen", "rows", "whole"}`, continued by the next pull - count `{"asks", "count"}`, the roles the last whole read listed, for - the 10% guard + count `{"asks", "count"}`, the roles the last read that was not a + sweep listed, for the 10% guard + failing `{"since", "reason"}`, kept by the Source (`record_failure`); + the guard gives way once `since` is `Limits.stale_days` old + remote_unasked "1" when the reach wanted remote and the read could not + ask for it (`Listing.remote_unasked`), else "" facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL` swept "1" when the last stamped listing finished a sweep carried over from an earlier pull (`Listing.swept`), else "". Offset paging @@ -87,10 +89,16 @@ DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "count #: How long a board's `facets` are kept before they are asked again. FACETS_TTL = timedelta(days=1) -#: A filtered or `empty="suspect"` read listing under this share of what the -#: same read listed last time counts as a failure. +#: A read listing under this share of what the same read listed last time +#: counts as a failure (`read`). SUSPECT_SHARE = 0.1 +#: The last count from which the guard holds every read. A read that is +#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from +#: a count of 1, as it was before every read was; a whole read of a board +#: that listed a handful of roles can drop to none by itself. +SUSPECT_MIN = 10 + #: The keys of a row that an `Item` carries in its own fields, not in `raw`. ITEM_KEYS = ("source_id", "title", "url", "description", "posted") @@ -105,8 +113,8 @@ class TooLarge(ValueError): class Suspect(Exception): - """A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE` - of what it listed last time.""" + """A read that listed under `SUSPECT_SHARE` of what it listed last time + (see `read`).""" class MissingKey(ValueError): @@ -115,12 +123,15 @@ class MissingKey(ValueError): @dataclass(frozen=True, slots=True) class Limits: - """What one Source allows a read: pages, the pause between requests, and - the detail reads a pull.""" + """What one Source allows a read: pages, the pause between requests, the + detail reads a pull, and the days after which the 10% guard accepts the + answer it keeps refusing (`stale_days`, the pull's `STALE_DAYS`; None, + for a Source run outside a pull, never).""" max_pages: int = MAX_PAGES page_delay: float = PAGE_DELAY detail_cap: int = DETAIL_CAP + stale_days: int | None = None @dataclass(slots=True) @@ -134,7 +145,9 @@ class Listing: neither are the whole board. `swept` says a complete listing finished a sweep carried over from an earlier pull: `ids` were read across pulls, and roles closing meanwhile may have shifted an offset-paged host's later - roles back past the cursor, so `ids` can miss a role that is still open.""" + roles back past the cursor, so `ids` can miss a role that is still open. + `remote_unasked` says a filtered read was meant to ask for remote roles + as well and could not, so its `ids` say nothing of the remote roles.""" rows: list[Row] complete: bool @@ -144,12 +157,14 @@ class Listing: asks: str = "" narrowed: bool = False swept: bool = False + remote_unasked: bool = False # --------------------------------------------------------------- modules def declared(module: ModuleType) -> bool: - """Whether `module` declares `READS`, and so takes `ask`.""" + """Whether `module` declares `READS`. Every reader does, so the runner no + longer asks; `app.jobs.reach` still does.""" return isinstance(getattr(module, "READS", None), Reads) @@ -334,6 +349,33 @@ def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int: return count +def _wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None, + reach: Mapping[str, Any] | None) -> tuple[dict[str, Any], bool]: + """The dimensions `reach` and then the params set, params winning, before + they are narrowed to what the reader declares, and whether the read is + wide. A reach of more than `MAX_SERIES` countries on a `one` country + filter is dropped whole, which makes the read wide; a `remote` of False + asks nothing.""" + given = reach or {} + reached = _dimensions(module, given) + wide = bool(given.get("wide")) + if module.READS.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES: + reached = {} + wide = True + wanted = reached | _dimensions(module, dict(subscription_params or {})) + if wanted.get("remote") is False: + del wanted["remote"] + return wanted, wide + + +def remote_wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None = None, + reach: Mapping[str, Any] | None = None) -> bool: + """Whether the reach or the params want remote roles, whether or not the + reader can ask for them. `read_listing` compares it with what the series + asked (`Listing.remote_unasked`).""" + return _wanted(module, subscription_params, reach)[0].get("remote") is True + + def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None, reach: Mapping[str, Any] | None = None, *, keys: Mapping[str, Any] | None = None) -> Ask: @@ -346,7 +388,8 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = dropped. A `remote` of False asks nothing. A search's params go whole into `Ask.params`; a reader with `READS.key` gets the resolved key under `Ask.params["key"]` (see `resolve_key`, which raises when it is missing). - A module without `READS` gets `Ask()`. + A remote the reader cannot filter on is dropped here; `remote_wanted` + still says it was wanted. Fetch terms (`reach["terms"]`) go only to a board reader that declares `query="one"`, and only on a wide read: `reach["wide"]` (a company set @@ -358,19 +401,10 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = whole, since remote without its countries would read only the remote roles; over it with terms alone, the terms go too and the board is read unfiltered.""" - if not declared(module): - return Ask() reads: Reads = module.READS params = dict(subscription_params or {}) given = reach or {} - reached = _dimensions(module, given) - wide = bool(given.get("wide")) - if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES: - reached = {} - wide = True - wanted = reached | _dimensions(module, params) - if wanted.get("remote") is False: - del wanted["remote"] + wanted, wide = _wanted(module, params, given) declared_now = reads.filters.declared() wanted = {name: value for name, value in wanted.items() if name in declared_now} terms = _terms(given.get("terms")) @@ -400,7 +434,7 @@ def narrowed(module: ModuleType, ask: Ask) -> bool: query. Such a read is never the whole board, nor the board within its places, so it closes nothing. A search's query is what the search is, and does not count.""" - board = not declared(module) or module.READS.scope == "board" + board = module.READS.scope == "board" return bool(ask.terms) or (board and bool(ask.query)) @@ -417,10 +451,7 @@ def series(module: ModuleType, ask: Ask, remote part beside places is then left out. Each of those is then asked once per fetch term (`ask.terms`), with the term as `query`: the host intersects a keyword with its facets, as the survey found on every - host, so the union by id is "one of the terms, in these places". `[ask]` - for a module without `READS`.""" - if not declared(module): - return [ask] + host, so the union by id is "one of the terms, in these places".""" if ask.remote and (ask.countries or ask.places): parts = [*_series(module, ask.model_copy(update={"remote": None}), facets), *_series(module, ask.model_copy(update={"countries": (), "places": ()}), @@ -450,8 +481,8 @@ def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str] known = known_facets.get(dimension) or {} mapped = tuple(dict.fromkeys(known[v] for v in values if v in known)) if not mapped: - log.info("readers: %s knows none of %s %s", module.NAME, dimension, - ", ".join(values)) + log.warning("readers: %s knows none of %s %s; nothing asked", module.NAME, + dimension, ", ".join(values)) return [] if dimension == "country": countries = mapped @@ -497,7 +528,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask], for index, ask in enumerate(asks): if index: time.sleep(limits.page_delay) - url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug) + url = module.feed_url(slug, ask) try: body = get(url) except TooLarge: @@ -514,7 +545,8 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask], def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *, limits: Limits | None = None, sweep: Mapping[str, Any] | None = None, can_sweep: bool = False, - facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing: + facets: Mapping[str, Mapping[str, str]] | None = None, + remote: bool = False) -> Listing: """One target's listing, read and parsed. Raises on any failure of the first request, and on a reader that does not understand an answer. @@ -532,22 +564,35 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None its series and cursor; the sweep is complete when it reaches the end, and `ids` then hold every id it listed, and the listing is `swept`. A read at the reader's `ceiling` is not swept: the host answers nothing - past it.""" + past it. + + A filtered read that asks no series, because an `ids` dimension names + no value the host's facets know, asked nothing, which is no evidence of + anything: it is incomplete and closes nothing. `filtered` is a board's + only: a search's places are what the search is, as its query is. + `remote` says the reach or the params wanted remote roles; when no series + asked for them (a reader without a remote filter, or one whose facets + state none beside places) the listing is `remote_unasked`.""" limits = limits or Limits() ask = ask or Ask() + reads: Reads = module.READS asks = series(module, ask, facets) - is_filtered = filtered(ask) + is_filtered = filtered(ask) and reads.scope == "board" by_words = narrowed(module, ask) + unasked = (is_filtered and (remote or bool(ask.remote)) + and not any(a.remote or a.remote_ids for a in asks)) key = fingerprint(asks) if not asks: - return Listing([], True, [], is_filtered, None, key, by_words) + log.warning("readers: %s:%s: the filter maps to nothing the host knows; " + "nothing asked, not closing", module.NAME, slug) + return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked) if not hasattr(module, "page"): - rows = _whole(module, slug, get, asks, limits) - return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words) + whole_rows = _whole(module, slug, get, asks, limits) + return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None, + key, by_words, remote_unasked=unasked) - reads: Reads | None = module.READS if declared(module) else None - never = reads is not None and reads.complete == "never" - ceiling = reads.ceiling if reads is not None else None + never = reads.complete == "never" + ceiling = reads.ceiling start, cursor, seen, earlier, whole = 0, None, set(), [], True resumed = bool(sweep and sweep.get("asks") == key) if sweep and resumed: @@ -568,10 +613,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None if pages: time.sleep(limits.page_delay) try: - if reads is not None: - data, results, total, cursor = module.page(get, slug, cursor, asks[index]) - else: - data, results, total, cursor = module.page(get, slug, cursor) + data, results, total, cursor = module.page(get, slug, cursor, asks[index]) except httpx.HTTPStatusError as exc: if not pages or exc.response.status_code != 429: raise @@ -603,15 +645,17 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None listed = list(dict.fromkeys(earlier + list(rows))) if not stop: return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words, - resumed and whole) + resumed and whole, unasked) log.warning("readers: %s:%s: stopped after %d pages (%s); not complete", module.NAME, slug, pages, stop) at_ceiling = ceiling is not None and len(seen) >= ceiling if not can_sweep or never or at_ceiling or not _storable(cursor): - return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words) + return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words, + remote_unasked=unasked) keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen), "rows": listed, "whole": whole} - return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words) + return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words, + remote_unasked=unasked) def _storable(cursor: Any) -> bool: @@ -637,7 +681,7 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask, """The reader's `facets` when `ask` sets a dimension it declares `ids` (a country, a place, or remote), kept in `state[key]` for `FACETS_TTL`; `{}` otherwise.""" - if not declared(module) or not hasattr(module, "facets"): + if not hasattr(module, "facets"): return {} ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"] if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"): @@ -659,46 +703,79 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask, def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *, - limits: Limits | None = None, can_sweep: bool = True) -> Listing: + limits: Limits | None = None, can_sweep: bool = True, remote: bool = False) -> Listing: """`read_listing` for one target with its state under `:...:`: - the facets it needs, the sweep it continues, and the 10% guard. A read - that is filtered, narrowed by terms, or whose reader declares - `empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same - read listed last time (the same `fingerprint`, so a term read is only - held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps - the rest once the rows are stored.""" + the facets it needs, the sweep it continues, and the 10% guard. Nothing + is written here but the facets; `stamp` keeps the rest once the rows are + stored. + + The guard holds every read that is not a sweep: one that listed under + `SUSPECT_SHARE` of what the same read listed last time (the same + `fingerprint`, so a term read is only held to a term read with the same + terms and places) raises `Suspect`, and the Source records the target + as failing, so its roles stay open. A host whose answer changed shape + parses to no rows with a 200, and closing a board on that would lose it + whole. It holds from a last count of `SUSPECT_MIN`, or of 1 for a read + that is filtered, narrowed or of an `empty="suspect"` reader. A board + that really shrank is accepted once the target has been failing for + `limits.stale_days` (its `:failing:` record), and its + stamp then resets the count; a Recheck resets it at once + (`plugins.ats.keep_size`).""" + limits = limits or Limits() ask = ask or Ask() facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}") listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep, - sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets) - suspect = declared(module) and module.READS.empty == "suspect" + sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets, + remote=remote) + held = listing.filtered or listing.narrowed or module.READS.empty == "suspect" last = _load(state, f"{prefix}:count:{target}") count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0 - if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count + if (listing.sweep is None and count >= (1 if held else SUSPECT_MIN) and len(listing.ids) < SUSPECT_SHARE * count): - raise Suspect(f"listed {len(listing.ids)} roles, under " - f"{SUSPECT_SHARE:.0%} of the {count} it listed last time") + why = (f"listed {len(listing.ids)} roles, under " + f"{SUSPECT_SHARE:.0%} of the {count} it listed last time") + if _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days): + log.warning("readers: %s:%s: %s, accepted after %d days of failing", + module.NAME, slug, why, limits.stale_days) + return listing + raise Suspect(why) return listing +def _failing_for(state: Mapping[str, str], key: str, days: int | None) -> bool: + """Whether the failure kept under `key` began at least `days` ago; False + for `days` None or no record.""" + since = str(_load(state, key).get("since") or "") + if days is None or not since: + return False + try: + began = datetime.fromisoformat(since) + except ValueError: + return False + return datetime.now(UTC) - began >= timedelta(days=days) + + def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str, listing: Listing, ids: list[str]) -> None: """Record a read whose rows are stored: `ok` (the run's marker), `complete`, `ids` (the compound ids, what the pull closes against), `filtered`, `narrowed` (a read by terms, which the pull never closes - against), whether it finished a sweep from earlier pulls (`swept`), the + against), whether it finished a sweep from earlier pulls (`swept`), + whether a filtered read left remote unasked (`remote_unasked`), the sweep to continue or none, and the count for the 10% guard once a read - is whole.""" + is not a sweep. A read that asked nothing keeps the count it had.""" state[f"{prefix}:ok:{target}"] = run state[f"{prefix}:complete:{target}"] = "1" if listing.complete else "" state[f"{prefix}:swept:{target}"] = "1" if listing.swept else "" state[f"{prefix}:ids:{target}"] = json.dumps(ids) state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else "" state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else "" + state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else "" if listing.sweep is not None: state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep) - else: - state.pop(f"{prefix}:sweep:{target}", None) + return + state.pop(f"{prefix}:sweep:{target}", None) + if listing.asks != fingerprint([]): state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks, "count": len(listing.ids)}) @@ -766,7 +843,7 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[ if ignored := sorted(set(answer) - set(DETAIL_FIELDS)): log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS", name, slug, row["source_id"], ", ".join(ignored)) - taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment] + taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} if taken.get("location") and "places" not in taken: taken["places"] = [{"text": taken["location"], "remote": None}] filled[key] = taken @@ -776,12 +853,12 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[ for row in rows: key = compound(row) if key in filled: - out.append(row | {"description": ""} | filled[key]) # type: ignore[operator] + out.append(row | {"description": ""} | filled[key]) elif key in detailed: - out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator] + out.append(row | {"description": "", "location": "", "places": [], "remote": None}) else: - out.append(row | {"description": ""}) # type: ignore[operator] + out.append(row | {"description": ""}) if listed is not None: done &= listed failing &= listed @@ -791,8 +868,9 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[ # -------------------------------------------------------------------- raw def places_of(row: Mapping[str, Any]) -> list[Place]: - """A row's places: its `places`, or, for a row without (an undeclared - reader's), its `location` as the one place, split nothing.""" + """A row's places: its `places`, or, for a row without (a raw row stored + before readers returned places), its `location` as the one place, split + nothing.""" places = row.get("places") if not isinstance(places, list): location = str(row.get("location") or "").strip() diff --git a/tests/test_readers.py b/tests/test_readers.py index 7791581..49251f6 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -26,7 +26,7 @@ if str(ROOT) not in sys.path: from plugins import ats, readers # noqa: E402 from plugins.readers import _runner # noqa: E402 from plugins.readers import lever # noqa: E402 -from plugins.readers._contract import Ask # noqa: E402 +from plugins.readers._contract import Ask, Filters, Reads # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -253,7 +253,9 @@ def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType: module.slug_ok = lambda slug: slug.isalnum() module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" - module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list" + module.READS = Reads(scope="board", listing="whole", complete="whole", places="one", + text="detail" if detail is not None else "listing", robots="test") + module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list" module.parse = lambda body, slug: [ {"source_id": i, "title": f"Role {i}", "location": "", "department": "", "url": f"https://jobs.fake.example/{slug}/{i}", "posted": None, @@ -590,3 +592,131 @@ def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None: with _client(handler) as client, pytest.raises(ats._Refused): ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x") + + +# ------------------------------------------- the guard, asks, remote unasked + +def _listing_host(**reads: Any) -> Any: + """A declared whole-listing board `fake` whose feed lists `module.ids`, + whatever is asked; `reads` override its declaration.""" + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () + module.slug_ok = lambda slug: slug.isalnum() + module.allowed = lambda slug: lambda h: h == "jobs.fake.example" + module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" + module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing", + "complete": "whole", "places": "one", "robots": "test"} | reads)) + module.ids = [] + module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list" + module.parse = lambda body, slug: [ + {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}", + "places": [{"text": "Berlin", "remote": None}]} for i in json.loads(body)] + return module + + +def _listed(module: Any) -> Callable[..., bytes]: + return lambda url, **kwargs: json.dumps(module.ids).encode() + + +def _read_and_stamp(state: dict[str, str], module: Any, + limits: _runner.Limits | None = None) -> _runner.Listing: + listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), + limits=limits) + _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) + return listing + + +def test_an_unfiltered_whole_read_of_nothing_after_forty_is_suspect() -> None: + """A Greenhouse-style board, trusted, unfiltered: a changed answer that + parses to no rows is refused rather than read as a board that closed.""" + module = _listing_host() + state: dict[str, str] = {} + module.ids = [str(n) for n in range(40)] + _read_and_stamp(state, module) + module.ids = [] + with pytest.raises(_runner.Suspect, match="listed 0 roles, under 10% of the 40"): + _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module)) + + +def test_a_whole_read_under_suspect_min_is_not_held() -> None: + module = _listing_host() + state: dict[str, str] = {} + module.ids = [str(n) for n in range(5)] + _read_and_stamp(state, module) + module.ids = [] + assert _read_and_stamp(state, module).ids == [] + assert json.loads(state["ats:count:fake:acme"])["count"] == 0 + + +def test_the_same_suspect_answer_is_accepted_after_stale_days() -> None: + module = _listing_host() + state: dict[str, str] = {} + module.ids = [str(n) for n in range(40)] + _read_and_stamp(state, module) + module.ids = ["1"] + failing = "ats:failing:fake:acme" + recent = (datetime.now(UTC) - timedelta(days=20)).isoformat(timespec="seconds") + state[failing] = json.dumps({"since": recent, "reason": "Suspect: listed 1 roles"}) + limits = _runner.Limits(page_delay=0, stale_days=21) + with pytest.raises(_runner.Suspect): + _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits) + + old = (datetime.now(UTC) - timedelta(days=22)).isoformat(timespec="seconds") + state[failing] = json.dumps({"since": old, "reason": "Suspect: listed 1 roles"}) + # Outside a pull (no `stale_days`) the guard never gives way. + with pytest.raises(_runner.Suspect): + _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module)) + assert _read_and_stamp(state, module, limits).ids == ["1"] + assert json.loads(state["ats:count:fake:acme"])["count"] == 1 + + +def test_keep_size_resets_the_guards_count_unless_told_not_to() -> None: + state = {"ats:count:fake:acme": json.dumps({"asks": "[]", "count": 40})} + ats.keep_size(state, "fake:acme", 40, reset=False) + assert "ats:count:fake:acme" in state + ats.keep_size(state, "fake:acme", 3) + assert "ats:count:fake:acme" not in state + assert state["ats:size:fake:acme"] == "3" + + +def test_a_filter_that_maps_to_no_host_id_asks_nothing_and_is_incomplete( + monkeypatch: pytest.MonkeyPatch) -> None: + module = _listing_host(filters=Filters(country="ids")) + module.facets = lambda get, slug: {"country": {}} + module.ids = ["1", "2"] + state: dict[str, str] = {} + listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), + Ask(countries=("DK",))) + assert (listing.rows, listing.ids, listing.complete, listing.filtered) == ( + [], [], False, True) + + # Through the plugin: stamped, but not complete, so the settle closes nothing. + monkeypatch.setitem(readers.HOSTS, "fake", module) + monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b'["1"]') + source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme", + "params": {"countries": ["DK"]}}], "run": "r"}) + assert list(source.fetch()) == [] + assert (source.state["ats:ok:fake:acme"], source.state["ats:complete:fake:acme"]) == ( + "r", "") + + +def test_a_filtered_read_that_cannot_ask_remote_says_so() -> None: + reach = {"countries": ["DK"], "remote": True} + blind = _listing_host(filters=Filters(country="many")) + ask = _runner.ask_for(blind, None, reach) + assert (ask.countries, ask.remote) == (("DK",), None) + assert _runner.remote_wanted(blind, None, reach) + listing = _runner.read_listing(blind, "acme", _listed(blind), ask, + remote=_runner.remote_wanted(blind, None, reach)) + assert listing.filtered and listing.remote_unasked + state: dict[str, str] = {} + _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) + assert state["ats:remote_unasked:fake:acme"] == "1" + + # A host that filters on remote asks it as a series of its own. + sees = _listing_host(filters=Filters(country="many", remote="many")) + ask = _runner.ask_for(sees, None, reach) + listing = _runner.read_listing(sees, "acme", _listed(sees), ask, remote=True) + assert listing.filtered and not listing.remote_unasked + # A params' remote of False asks nothing and wants nothing. + assert not _runner.remote_wanted(blind, {"remote": False}, reach) diff --git a/tests/test_runner.py b/tests/test_runner.py index 69fd80a..124cf4f 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -103,8 +103,6 @@ def test_ask_for_narrows_the_params_to_the_declared_filters() -> None: both = _whole(filters=Filters(country="many", remote="many")) ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True}) assert (ask.countries, ask.remote) == (("DE",), True) - # A module without READS reads the whole board. - assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask() def test_a_searchs_params_go_whole_into_the_ask_with_its_key( @@ -139,11 +137,12 @@ def test_ids_filters_are_mapped_through_the_facets() -> None: ask = Ask(countries=("DK", "XX")) [mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}}) assert mapped.countries == ("h-1",) - # None known: the host has nothing in reach, and the read is empty. + # None known: nothing is asked, which is no evidence, so the read is + # incomplete and closes nothing. assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == [] listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)), facets={"country": {}}) - assert (listing.rows, listing.complete, listing.filtered) == ([], True, True) + assert (listing.rows, listing.complete, listing.filtered) == ([], False, True) def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None: @@ -189,17 +188,6 @@ def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None: _runner.read_listing(module, "acme", web) -def test_a_module_without_reads_gets_the_older_signatures() -> None: - module: Any = types.ModuleType("old") - module.NAME = "old" - module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list" - module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] - listing = _runner.read_listing(module, "acme", - Web({"https://jobs.fake.example/acme/list": b'["1"]'}), - Ask(countries=("DK",))) - assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete - - def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None: pages = [["1", "2"], ["3"]] listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages), @@ -333,9 +321,11 @@ def test_a_suspect_read_far_under_the_last_count_fails() -> None: web.answers["https://jobs.fake.example/acme/list"] = b'["1"]' with pytest.raises(_runner.Suspect, match="under 10% of the 20"): _runner.read(state, "ats", "t", module, "acme", web) - # A reader whose empty answer is trusted reads it as it is. + # A reader whose empty answer is trusted is held too, from a count of + # `SUSPECT_MIN`. module.READS = module.READS.model_copy(update={"empty": "trusted"}) - assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"] + with pytest.raises(_runner.Suspect): + _runner.read(state, "ats", "t", module, "acme", web) def test_the_guard_compares_only_reads_that_asked_the_same() -> None: