Guard every read that is not a sweep, treat a read that asks nothing as incomplete, and mark a filtered read that could not ask remote, so no board loses its roles to a changed answer or a filter it could not send

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:53:47 +02:00
1 parent 4e827665dd
commit 36bc639384
4 files changed
+329 -113

No files matched your search

+41 -23
View File
@@ -16,7 +16,9 @@ The allow-list and the classifier are ported from `openings.go` and
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is read independently; one that fails is logged, its slug is. Every board is read independently; one that fails is logged,
recorded as failing under `ats:failing:<target>`, and skipped. A board's recorded as failing under `ats:failing:<target>`, and skipped, and so is one
whose read lists under a tenth of what it listed last time (`_runner.read`'s
guard), until it has failed that way for the pull's `stale_days`. A board's
filter is the `reach` option the pull hands it (the agents', see filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries": `app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`) win, narrowed to what its reader declares ["DK"], "remote": true}`) win, narrowed to what its reader declares
@@ -26,7 +28,8 @@ the whole board: it prunes no per-role state.
A board's size (`board_size`) is set once, when the board is added A board's size (`board_size`) is set once, when the board is added
(`app.jobs.sources.set_board`) or rechecked, and kept under (`app.jobs.sources.set_board`) or rechecked, and kept under
`ats:size:<target>` with the day it was checked under `ats:size:<target>` with the day it was checked under
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small `ats:sizechecked:<target>` (`keep_size`, which also resets the guard's count,
so a Recheck accepts a board that shrank); the reach reads it to leave small
boards whole. A pull does not rewrite it. A board with no size yet is sized boards whole. A pull does not rewrite it. A board with no size yet is sized
in its batch before it is read: its reach then says `unsized` and the size in its batch before it is read: its reach then says `unsized` and the size
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
@@ -106,8 +109,9 @@ _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")} name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
def _limits() -> _runner.Limits: def _limits(stale_days: int | None = None) -> _runner.Limits:
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP) return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP,
stale_days=stale_days)
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
@@ -339,8 +343,7 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
module = readers.HOSTS[host] module = readers.HOSTS[host]
get = _getter(client, host, slug) get = _getter(client, host, slug)
if (_runner.declared(module) and module.READS.listing == "paged" if module.READS.listing == "paged" and module.READS.complete == "stated":
and module.READS.complete == "stated"):
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask()) _data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
if isinstance(total, int): if isinstance(total, int):
return total return total
@@ -348,11 +351,18 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
return len(_runner.read_listing(module, slug, get, limits=limits).ids) return len(_runner.read_listing(module, slug, get, limits=limits).ids)
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None: def keep_size(state: Any, target: str, size: int, *, today: str = "",
reset: bool = True) -> None:
"""Keep board `target`'s size and the day it was checked (ISO, """Keep board `target`'s size and the day it was checked (ISO,
`today`, else today in UTC).""" `today`, else today in UTC). With `reset`, the default and what a
Recheck does, the 10% guard's count goes too (`ats:count:<target>`), so
the next read of a board that really shrank is taken as it is instead of
failing as suspect; the plugin sizing a board as it reads it passes
False, since its count is the read it just stamped."""
state[f"ats:size:{target}"] = str(size) state[f"ats:size:{target}"] = str(size)
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat() state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
if reset:
state.pop(f"ats:count:{target}", None)
failure_reason = _runner.failure_reason failure_reason = _runner.failure_reason
@@ -400,14 +410,15 @@ class ATSSource(Source):
raise ValueError(f"{target!r} is not host:slug for a known ATS host") raise ValueError(f"{target!r} is not host:slug for a known ATS host")
module = readers.HOSTS[host] module = readers.HOSTS[host]
get = _getter(client, host, slug) get = _getter(client, host, slug)
ask, unsized = self._ask(client, target, module) ask, unsized, remote = self._ask(client, target, module)
stale_days = self.options.get("stale_days")
listing = _runner.read(self.state, "ats", target, module, slug, get, ask, listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
limits=_limits()) limits=_limits(int(stale_days) if stale_days else None),
remote=remote)
except Exception as exc: except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc) log.warning("ats: %s failed (%s)", target, exc)
_runner.record_failure(self.state, failing, exc) _runner.record_failure(self.state, failing, exc)
return return
self.state.pop(failing, None)
def compound(source_id: str) -> str: def compound(source_id: str) -> str:
# Matches `roles.source_id`, which `Store.mark_closed` compares. # Matches `roles.source_id`, which `Store.mark_closed` compares.
@@ -435,11 +446,13 @@ class ATSSource(Source):
yield item_of(row, target, label) yield item_of(row, target, label)
# Stamped only once every row went out, so a board that blew up half # Stamped only once every row went out, so a board that blew up half
# way through is neither marked ok nor has a partial id list treated # way through is neither marked ok nor has a partial id list treated
# as the whole of it. # as the whole of it, and it stays failing until then: a row that
# fails to store (`app.jobs.pull.run_batch`) leaves the record in place.
_runner.stamp(self.state, "ats", target, run_marker, listing, _runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids]) [compound(i) for i in listing.ids])
self.state.pop(failing, None)
if unsized and listing.complete and not (listing.filtered or listing.narrowed): if unsized and listing.complete and not (listing.filtered or listing.narrowed):
keep_size(self.state, target, len(listing.ids)) keep_size(self.state, target, len(listing.ids), reset=False)
if detail_sets is not None: if detail_sets is not None:
# A complete listing drops the ids it no longer lists. # A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False): for key, ids in zip(detail_keys(target), detail_sets, strict=False):
@@ -447,9 +460,11 @@ class ATSSource(Source):
self.state[key] = json.dumps(sorted(ids)) self.state[key] = json.dumps(sorted(ids))
def _ask(self, client: httpx.Client, target: str, def _ask(self, client: httpx.Client, target: str,
module: Any) -> tuple[readers.Ask, bool]: module: Any) -> tuple[readers.Ask, bool, bool]:
"""What `target` is asked for, from the reach its batch handed it and """What `target` is asked for, from the reach its batch handed it and
its params, and whether it still needs a size. A board with no size its params, whether it still needs a size, and whether that reach
and params want remote roles (`_runner.remote_wanted`, which the
runner compares with what it could ask). A board with no size
yet (`unsized` in its reach) whose reach would narrow the read is yet (`unsized` in its reach) whose reach would narrow the read is
sized first (`board_size`, kept by `keep_size`), and read whole under sized first (`board_size`, kept by `keep_size`), and read whole under
`small` roles. One whose read is whole anyway is sized by that read, `small` roles. One whose read is whole anyway is sized by that read,
@@ -457,16 +472,18 @@ class ATSSource(Source):
given = self.options.get("reach") given = self.options.get("reach")
params, keys = self.params_for(target), self.options.get("keys") params, keys = self.params_for(target), self.options.get("keys")
if not isinstance(given, dict) or not given.get("unsized"): if not isinstance(given, dict) or not given.get("unsized"):
return _runner.ask_for(module, params, given, keys=keys), False return (_runner.ask_for(module, params, given, keys=keys), False,
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} _runner.remote_wanted(module, params, given))
ask = _runner.ask_for(module, params, reach or None, keys=keys) reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None
ask = _runner.ask_for(module, params, reach, keys=keys)
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)): if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
return ask, True return ask, True, _runner.remote_wanted(module, params, reach)
size = board_size(target, client=client) size = board_size(target, client=client)
keep_size(self.state, target, size) keep_size(self.state, target, size, reset=False)
if size < int(given.get("small") or 0): if size < int(given.get("small") or 0):
reach = None
ask = _runner.ask_for(module, params, None, keys=keys) ask = _runner.ask_for(module, params, None, keys=keys)
return ask, False return ask, False, _runner.remote_wanted(module, params, reach)
# ------------------------------------------------------ the per-role reads # ------------------------------------------------------ the per-role reads
@@ -567,7 +584,8 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
def item_of(row: Row, target: str, label: str) -> Item: def item_of(row: Row, target: str, label: str) -> Item:
"""One row of board `target` as the `Item` the store takes: its id """One row of board `target` as the `Item` the store takes: its id
compound, `host:slug:id`, as `roles.source_id` has it.""" compound, `host:slug:id`, as `roles.source_id` has it. A row without a
date is dated now, as `Item` itself would date it."""
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
description = row.get("description") or "" description = row.get("description") or ""
return Item( return Item(
@@ -576,7 +594,7 @@ def item_of(row: Row, target: str, label: str) -> Item:
title=row["title"], title=row["title"],
excerpt=description[:300], excerpt=description[:300],
body=description, body=description,
published_at=row.get("posted"), published_at=row.get("posted") or datetime.now(UTC),
subscription=label, subscription=label,
kind="role", kind="role",
raw=_runner.raw_of(row) | { raw=_runner.raw_of(row) | {
+149 -71
View File
@@ -5,23 +5,25 @@ A reader says how its host answers; this module reads it: the guarded fetch
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves (`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
to (`ask_for`) and its series (`series`), the listing whole or paged, one to (`ask_for`) and its series (`series`), the listing whole or paged, one
series per `one`-filter value unioned by id (`read_listing`), completeness, series per `one`-filter value unioned by id (`read_listing`), completeness,
the sweep of an oversized paged board across pulls, the 10% guard on a the sweep of an oversized paged board across pulls, the 10% guard on every
filtered or `empty="suspect"` read (`read`), the per-role detail reads read that is not a sweep (`read`), the per-role detail reads
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
`plugins/search.py` are thin `Source` classes over it: they pass their own `plugins/search.py` are thin `Source` classes over it: they pass their own
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
as they were. as they were. Every reader declares `READS`; a module without it is refused
by the package before it gets here.
A module without `READS` is read with the older signatures, `feed_url(slug)`
and `page(get, slug, cursor)`, and never gets an `Ask`.
State keys, under the Source's prefix and the target (`ats:sweep:<target>`): State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor", sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
"seen", "rows", "whole"}`, continued by the next pull "seen", "rows", "whole"}`, continued by the next pull
count `{"asks", "count"}`, the roles the last whole read listed, for count `{"asks", "count"}`, the roles the last read that was not a
the 10% guard sweep listed, for the 10% guard
failing `{"since", "reason"}`, kept by the Source (`record_failure`);
the guard gives way once `since` is `Limits.stale_days` old
remote_unasked "1" when the reach wanted remote and the read could not
ask for it (`Listing.remote_unasked`), else ""
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL` facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
swept "1" when the last stamped listing finished a sweep carried over swept "1" when the last stamped listing finished a sweep carried over
from an earlier pull (`Listing.swept`), else "". Offset paging from an earlier pull (`Listing.swept`), else "". Offset paging
@@ -87,10 +89,16 @@ DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "count
#: How long a board's `facets` are kept before they are asked again. #: How long a board's `facets` are kept before they are asked again.
FACETS_TTL = timedelta(days=1) FACETS_TTL = timedelta(days=1)
#: A filtered or `empty="suspect"` read listing under this share of what the #: A read listing under this share of what the same read listed last time
#: same read listed last time counts as a failure. #: counts as a failure (`read`).
SUSPECT_SHARE = 0.1 SUSPECT_SHARE = 0.1
#: The last count from which the guard holds every read. A read that is
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
#: a count of 1, as it was before every read was; a whole read of a board
#: that listed a handful of roles can drop to none by itself.
SUSPECT_MIN = 10
#: The keys of a row that an `Item` carries in its own fields, not in `raw`. #: The keys of a row that an `Item` carries in its own fields, not in `raw`.
ITEM_KEYS = ("source_id", "title", "url", "description", "posted") ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
@@ -105,8 +113,8 @@ class TooLarge(ValueError):
class Suspect(Exception): class Suspect(Exception):
"""A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE` """A read that listed under `SUSPECT_SHARE` of what it listed last time
of what it listed last time.""" (see `read`)."""
class MissingKey(ValueError): class MissingKey(ValueError):
@@ -115,12 +123,15 @@ class MissingKey(ValueError):
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
class Limits: class Limits:
"""What one Source allows a read: pages, the pause between requests, and """What one Source allows a read: pages, the pause between requests, the
the detail reads a pull.""" detail reads a pull, and the days after which the 10% guard accepts the
answer it keeps refusing (`stale_days`, the pull's `STALE_DAYS`; None,
for a Source run outside a pull, never)."""
max_pages: int = MAX_PAGES max_pages: int = MAX_PAGES
page_delay: float = PAGE_DELAY page_delay: float = PAGE_DELAY
detail_cap: int = DETAIL_CAP detail_cap: int = DETAIL_CAP
stale_days: int | None = None
@dataclass(slots=True) @dataclass(slots=True)
@@ -134,7 +145,9 @@ class Listing:
neither are the whole board. `swept` says a complete listing finished a neither are the whole board. `swept` says a complete listing finished a
sweep carried over from an earlier pull: `ids` were read across pulls, sweep carried over from an earlier pull: `ids` were read across pulls,
and roles closing meanwhile may have shifted an offset-paged host's later and roles closing meanwhile may have shifted an offset-paged host's later
roles back past the cursor, so `ids` can miss a role that is still open.""" roles back past the cursor, so `ids` can miss a role that is still open.
`remote_unasked` says a filtered read was meant to ask for remote roles
as well and could not, so its `ids` say nothing of the remote roles."""
rows: list[Row] rows: list[Row]
complete: bool complete: bool
@@ -144,12 +157,14 @@ class Listing:
asks: str = "" asks: str = ""
narrowed: bool = False narrowed: bool = False
swept: bool = False swept: bool = False
remote_unasked: bool = False
# --------------------------------------------------------------- modules # --------------------------------------------------------------- modules
def declared(module: ModuleType) -> bool: def declared(module: ModuleType) -> bool:
"""Whether `module` declares `READS`, and so takes `ask`.""" """Whether `module` declares `READS`. Every reader does, so the runner no
longer asks; `app.jobs.reach` still does."""
return isinstance(getattr(module, "READS", None), Reads) return isinstance(getattr(module, "READS", None), Reads)
@@ -334,6 +349,33 @@ def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
return count return count
def _wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None,
reach: Mapping[str, Any] | None) -> tuple[dict[str, Any], bool]:
"""The dimensions `reach` and then the params set, params winning, before
they are narrowed to what the reader declares, and whether the read is
wide. A reach of more than `MAX_SERIES` countries on a `one` country
filter is dropped whole, which makes the read wide; a `remote` of False
asks nothing."""
given = reach or {}
reached = _dimensions(module, given)
wide = bool(given.get("wide"))
if module.READS.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
reached = {}
wide = True
wanted = reached | _dimensions(module, dict(subscription_params or {}))
if wanted.get("remote") is False:
del wanted["remote"]
return wanted, wide
def remote_wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
reach: Mapping[str, Any] | None = None) -> bool:
"""Whether the reach or the params want remote roles, whether or not the
reader can ask for them. `read_listing` compares it with what the series
asked (`Listing.remote_unasked`)."""
return _wanted(module, subscription_params, reach)[0].get("remote") is True
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None, def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
reach: Mapping[str, Any] | None = None, *, reach: Mapping[str, Any] | None = None, *,
keys: Mapping[str, Any] | None = None) -> Ask: keys: Mapping[str, Any] | None = None) -> Ask:
@@ -346,7 +388,8 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
dropped. A `remote` of False asks nothing. A search's params go whole into dropped. A `remote` of False asks nothing. A search's params go whole into
`Ask.params`; a reader with `READS.key` gets the resolved key under `Ask.params`; a reader with `READS.key` gets the resolved key under
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing). `Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
A module without `READS` gets `Ask()`. A remote the reader cannot filter on is dropped here; `remote_wanted`
still says it was wanted.
Fetch terms (`reach["terms"]`) go only to a board reader that declares Fetch terms (`reach["terms"]`) go only to a board reader that declares
`query="one"`, and only on a wide read: `reach["wide"]` (a company set `query="one"`, and only on a wide read: `reach["wide"]` (a company set
@@ -358,19 +401,10 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
whole, since remote without its countries would read only the remote whole, since remote without its countries would read only the remote
roles; over it with terms alone, the terms go too and the board is read roles; over it with terms alone, the terms go too and the board is read
unfiltered.""" unfiltered."""
if not declared(module):
return Ask()
reads: Reads = module.READS reads: Reads = module.READS
params = dict(subscription_params or {}) params = dict(subscription_params or {})
given = reach or {} given = reach or {}
reached = _dimensions(module, given) wanted, wide = _wanted(module, params, given)
wide = bool(given.get("wide"))
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
reached = {}
wide = True
wanted = reached | _dimensions(module, params)
if wanted.get("remote") is False:
del wanted["remote"]
declared_now = reads.filters.declared() declared_now = reads.filters.declared()
wanted = {name: value for name, value in wanted.items() if name in declared_now} wanted = {name: value for name, value in wanted.items() if name in declared_now}
terms = _terms(given.get("terms")) terms = _terms(given.get("terms"))
@@ -400,7 +434,7 @@ def narrowed(module: ModuleType, ask: Ask) -> bool:
query. Such a read is never the whole board, nor the board within its query. Such a read is never the whole board, nor the board within its
places, so it closes nothing. A search's query is what places, so it closes nothing. A search's query is what
the search is, and does not count.""" the search is, and does not count."""
board = not declared(module) or module.READS.scope == "board" board = module.READS.scope == "board"
return bool(ask.terms) or (board and bool(ask.query)) return bool(ask.terms) or (board and bool(ask.query))
@@ -417,10 +451,7 @@ def series(module: ModuleType, ask: Ask,
remote part beside places is then left out. Each of those is then asked remote part beside places is then left out. Each of those is then asked
once per fetch term (`ask.terms`), with the term as `query`: the host once per fetch term (`ask.terms`), with the term as `query`: the host
intersects a keyword with its facets, as the survey found on every intersects a keyword with its facets, as the survey found on every
host, so the union by id is "one of the terms, in these places". `[ask]` host, so the union by id is "one of the terms, in these places"."""
for a module without `READS`."""
if not declared(module):
return [ask]
if ask.remote and (ask.countries or ask.places): if ask.remote and (ask.countries or ask.places):
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets), parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
*_series(module, ask.model_copy(update={"countries": (), "places": ()}), *_series(module, ask.model_copy(update={"countries": (), "places": ()}),
@@ -450,8 +481,8 @@ def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]
known = known_facets.get(dimension) or {} known = known_facets.get(dimension) or {}
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known)) mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
if not mapped: if not mapped:
log.info("readers: %s knows none of %s %s", module.NAME, dimension, log.warning("readers: %s knows none of %s %s; nothing asked", module.NAME,
", ".join(values)) dimension, ", ".join(values))
return [] return []
if dimension == "country": if dimension == "country":
countries = mapped countries = mapped
@@ -497,7 +528,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
for index, ask in enumerate(asks): for index, ask in enumerate(asks):
if index: if index:
time.sleep(limits.page_delay) time.sleep(limits.page_delay)
url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug) url = module.feed_url(slug, ask)
try: try:
body = get(url) body = get(url)
except TooLarge: except TooLarge:
@@ -514,7 +545,8 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *, def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None, limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
can_sweep: bool = False, can_sweep: bool = False,
facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing: facets: Mapping[str, Mapping[str, str]] | None = None,
remote: bool = False) -> Listing:
"""One target's listing, read and parsed. Raises on any failure of the """One target's listing, read and parsed. Raises on any failure of the
first request, and on a reader that does not understand an answer. first request, and on a reader that does not understand an answer.
@@ -532,22 +564,35 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
its series and cursor; the sweep is complete when it reaches the end, its series and cursor; the sweep is complete when it reaches the end,
and `ids` then hold every id it listed, and the listing is `swept`. A and `ids` then hold every id it listed, and the listing is `swept`. A
read at the reader's `ceiling` is not swept: the host answers nothing read at the reader's `ceiling` is not swept: the host answers nothing
past it.""" past it.
A filtered read that asks no series, because an `ids` dimension names
no value the host's facets know, asked nothing, which is no evidence of
anything: it is incomplete and closes nothing. `filtered` is a board's
only: a search's places are what the search is, as its query is.
`remote` says the reach or the params wanted remote roles; when no series
asked for them (a reader without a remote filter, or one whose facets
state none beside places) the listing is `remote_unasked`."""
limits = limits or Limits() limits = limits or Limits()
ask = ask or Ask() ask = ask or Ask()
reads: Reads = module.READS
asks = series(module, ask, facets) asks = series(module, ask, facets)
is_filtered = filtered(ask) is_filtered = filtered(ask) and reads.scope == "board"
by_words = narrowed(module, ask) by_words = narrowed(module, ask)
unasked = (is_filtered and (remote or bool(ask.remote))
and not any(a.remote or a.remote_ids for a in asks))
key = fingerprint(asks) key = fingerprint(asks)
if not asks: if not asks:
return Listing([], True, [], is_filtered, None, key, by_words) log.warning("readers: %s:%s: the filter maps to nothing the host knows; "
"nothing asked, not closing", module.NAME, slug)
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
if not hasattr(module, "page"): if not hasattr(module, "page"):
rows = _whole(module, slug, get, asks, limits) whole_rows = _whole(module, slug, get, asks, limits)
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words) return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
key, by_words, remote_unasked=unasked)
reads: Reads | None = module.READS if declared(module) else None never = reads.complete == "never"
never = reads is not None and reads.complete == "never" ceiling = reads.ceiling
ceiling = reads.ceiling if reads is not None else None
start, cursor, seen, earlier, whole = 0, None, set(), [], True start, cursor, seen, earlier, whole = 0, None, set(), [], True
resumed = bool(sweep and sweep.get("asks") == key) resumed = bool(sweep and sweep.get("asks") == key)
if sweep and resumed: if sweep and resumed:
@@ -568,10 +613,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
if pages: if pages:
time.sleep(limits.page_delay) time.sleep(limits.page_delay)
try: try:
if reads is not None:
data, results, total, cursor = module.page(get, slug, cursor, asks[index]) data, results, total, cursor = module.page(get, slug, cursor, asks[index])
else:
data, results, total, cursor = module.page(get, slug, cursor)
except httpx.HTTPStatusError as exc: except httpx.HTTPStatusError as exc:
if not pages or exc.response.status_code != 429: if not pages or exc.response.status_code != 429:
raise raise
@@ -603,15 +645,17 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
listed = list(dict.fromkeys(earlier + list(rows))) listed = list(dict.fromkeys(earlier + list(rows)))
if not stop: if not stop:
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words, return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
resumed and whole) resumed and whole, unasked)
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete", log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
module.NAME, slug, pages, stop) module.NAME, slug, pages, stop)
at_ceiling = ceiling is not None and len(seen) >= ceiling at_ceiling = ceiling is not None and len(seen) >= ceiling
if not can_sweep or never or at_ceiling or not _storable(cursor): if not can_sweep or never or at_ceiling or not _storable(cursor):
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words) return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words,
remote_unasked=unasked)
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen), keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
"rows": listed, "whole": whole} "rows": listed, "whole": whole}
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words) return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words,
remote_unasked=unasked)
def _storable(cursor: Any) -> bool: def _storable(cursor: Any) -> bool:
@@ -637,7 +681,7 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
"""The reader's `facets` when `ask` sets a dimension it declares `ids` """The reader's `facets` when `ask` sets a dimension it declares `ids`
(a country, a place, or remote), (a country, a place, or remote),
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise.""" kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
if not declared(module) or not hasattr(module, "facets"): if not hasattr(module, "facets"):
return {} return {}
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"] ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"): if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
@@ -659,26 +703,56 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType, def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
slug: str, get: Get, ask: Ask | None = None, *, slug: str, get: Get, ask: Ask | None = None, *,
limits: Limits | None = None, can_sweep: bool = True) -> Listing: limits: Limits | None = None, can_sweep: bool = True, remote: bool = False) -> Listing:
"""`read_listing` for one target with its state under `<prefix>:...:<target>`: """`read_listing` for one target with its state under `<prefix>:...:<target>`:
the facets it needs, the sweep it continues, and the 10% guard. A read the facets it needs, the sweep it continues, and the 10% guard. Nothing
that is filtered, narrowed by terms, or whose reader declares is written here but the facets; `stamp` keeps the rest once the rows are
`empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same stored.
read listed last time (the same `fingerprint`, so a term read is only
held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps The guard holds every read that is not a sweep: one that listed under
the rest once the rows are stored.""" `SUSPECT_SHARE` of what the same read listed last time (the same
`fingerprint`, so a term read is only held to a term read with the same
terms and places) raises `Suspect`, and the Source records the target
as failing, so its roles stay open. A host whose answer changed shape
parses to no rows with a 200, and closing a board on that would lose it
whole. It holds from a last count of `SUSPECT_MIN`, or of 1 for a read
that is filtered, narrowed or of an `empty="suspect"` reader. A board
that really shrank is accepted once the target has been failing for
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
stamp then resets the count; a Recheck resets it at once
(`plugins.ats.keep_size`)."""
limits = limits or Limits()
ask = ask or Ask() ask = ask or Ask()
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}") facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep, listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets) sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
suspect = declared(module) and module.READS.empty == "suspect" remote=remote)
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
last = _load(state, f"{prefix}:count:{target}") last = _load(state, f"{prefix}:count:{target}")
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0 count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count if (listing.sweep is None and count >= (1 if held else SUSPECT_MIN)
and len(listing.ids) < SUSPECT_SHARE * count): and len(listing.ids) < SUSPECT_SHARE * count):
raise Suspect(f"listed {len(listing.ids)} roles, under " why = (f"listed {len(listing.ids)} roles, under "
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time") f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
if _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
module.NAME, slug, why, limits.stale_days)
return listing return listing
raise Suspect(why)
return listing
def _failing_for(state: Mapping[str, str], key: str, days: int | None) -> bool:
"""Whether the failure kept under `key` began at least `days` ago; False
for `days` None or no record."""
since = str(_load(state, key).get("since") or "")
if days is None or not since:
return False
try:
began = datetime.fromisoformat(since)
except ValueError:
return False
return datetime.now(UTC) - began >= timedelta(days=days)
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str, def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
@@ -686,19 +760,22 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
"""Record a read whose rows are stored: `ok` (the run's marker), """Record a read whose rows are stored: `ok` (the run's marker),
`complete`, `ids` (the compound ids, what the pull closes against), `complete`, `ids` (the compound ids, what the pull closes against),
`filtered`, `narrowed` (a read by terms, which the pull never closes `filtered`, `narrowed` (a read by terms, which the pull never closes
against), whether it finished a sweep from earlier pulls (`swept`), the against), whether it finished a sweep from earlier pulls (`swept`),
whether a filtered read left remote unasked (`remote_unasked`), the
sweep to continue or none, and the count for the 10% guard once a read sweep to continue or none, and the count for the 10% guard once a read
is whole.""" is not a sweep. A read that asked nothing keeps the count it had."""
state[f"{prefix}:ok:{target}"] = run state[f"{prefix}:ok:{target}"] = run
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else "" state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else "" state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
state[f"{prefix}:ids:{target}"] = json.dumps(ids) state[f"{prefix}:ids:{target}"] = json.dumps(ids)
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else "" state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else "" state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
if listing.sweep is not None: if listing.sweep is not None:
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep) state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
else: return
state.pop(f"{prefix}:sweep:{target}", None) state.pop(f"{prefix}:sweep:{target}", None)
if listing.asks != fingerprint([]):
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks, state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
"count": len(listing.ids)}) "count": len(listing.ids)})
@@ -766,7 +843,7 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)): if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS", log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
name, slug, row["source_id"], ", ".join(ignored)) name, slug, row["source_id"], ", ".join(ignored))
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment] taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
if taken.get("location") and "places" not in taken: if taken.get("location") and "places" not in taken:
taken["places"] = [{"text": taken["location"], "remote": None}] taken["places"] = [{"text": taken["location"], "remote": None}]
filled[key] = taken filled[key] = taken
@@ -776,12 +853,12 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
for row in rows: for row in rows:
key = compound(row) key = compound(row)
if key in filled: if key in filled:
out.append(row | {"description": ""} | filled[key]) # type: ignore[operator] out.append(row | {"description": ""} | filled[key])
elif key in detailed: elif key in detailed:
out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator] out.append(row | {"description": "", "location": "", "places": [],
"remote": None}) "remote": None})
else: else:
out.append(row | {"description": ""}) # type: ignore[operator] out.append(row | {"description": ""})
if listed is not None: if listed is not None:
done &= listed done &= listed
failing &= listed failing &= listed
@@ -791,8 +868,9 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
# -------------------------------------------------------------------- raw # -------------------------------------------------------------------- raw
def places_of(row: Mapping[str, Any]) -> list[Place]: def places_of(row: Mapping[str, Any]) -> list[Place]:
"""A row's places: its `places`, or, for a row without (an undeclared """A row's places: its `places`, or, for a row without (a raw row stored
reader's), its `location` as the one place, split nothing.""" before readers returned places), its `location` as the one place, split
nothing."""
places = row.get("places") places = row.get("places")
if not isinstance(places, list): if not isinstance(places, list):
location = str(row.get("location") or "").strip() location = str(row.get("location") or "").strip()
+132 -2
View File
@@ -26,7 +26,7 @@ if str(ROOT) not in sys.path:
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import _runner # noqa: E402 from plugins.readers import _runner # noqa: E402
from plugins.readers import lever # noqa: E402 from plugins.readers import lever # noqa: E402
from plugins.readers._contract import Ask # noqa: E402 from plugins.readers._contract import Ask, Filters, Reads # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -253,7 +253,9 @@ def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
module.slug_ok = lambda slug: slug.isalnum() module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list" module.READS = Reads(scope="board", listing="whole", complete="whole", places="one",
text="detail" if detail is not None else "listing", robots="test")
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [ module.parse = lambda body, slug: [
{"source_id": i, "title": f"Role {i}", "location": "", "department": "", {"source_id": i, "title": f"Role {i}", "location": "", "department": "",
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None, "url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
@@ -590,3 +592,131 @@ def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
with _client(handler) as client, pytest.raises(ats._Refused): with _client(handler) as client, pytest.raises(ats._Refused):
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x") ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
# ------------------------------------------- the guard, asks, remote unasked
def _listing_host(**reads: Any) -> Any:
"""A declared whole-listing board `fake` whose feed lists `module.ids`,
whatever is asked; `reads` override its declaration."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
"complete": "whole", "places": "one", "robots": "test"} | reads))
module.ids = []
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [
{"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
"places": [{"text": "Berlin", "remote": None}]} for i in json.loads(body)]
return module
def _listed(module: Any) -> Callable[..., bytes]:
return lambda url, **kwargs: json.dumps(module.ids).encode()
def _read_and_stamp(state: dict[str, str], module: Any,
limits: _runner.Limits | None = None) -> _runner.Listing:
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
return listing
def test_an_unfiltered_whole_read_of_nothing_after_forty_is_suspect() -> None:
"""A Greenhouse-style board, trusted, unfiltered: a changed answer that
parses to no rows is refused rather than read as a board that closed."""
module = _listing_host()
state: dict[str, str] = {}
module.ids = [str(n) for n in range(40)]
_read_and_stamp(state, module)
module.ids = []
with pytest.raises(_runner.Suspect, match="listed 0 roles, under 10% of the 40"):
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
def test_a_whole_read_under_suspect_min_is_not_held() -> None:
module = _listing_host()
state: dict[str, str] = {}
module.ids = [str(n) for n in range(5)]
_read_and_stamp(state, module)
module.ids = []
assert _read_and_stamp(state, module).ids == []
assert json.loads(state["ats:count:fake:acme"])["count"] == 0
def test_the_same_suspect_answer_is_accepted_after_stale_days() -> None:
module = _listing_host()
state: dict[str, str] = {}
module.ids = [str(n) for n in range(40)]
_read_and_stamp(state, module)
module.ids = ["1"]
failing = "ats:failing:fake:acme"
recent = (datetime.now(UTC) - timedelta(days=20)).isoformat(timespec="seconds")
state[failing] = json.dumps({"since": recent, "reason": "Suspect: listed 1 roles"})
limits = _runner.Limits(page_delay=0, stale_days=21)
with pytest.raises(_runner.Suspect):
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits)
old = (datetime.now(UTC) - timedelta(days=22)).isoformat(timespec="seconds")
state[failing] = json.dumps({"since": old, "reason": "Suspect: listed 1 roles"})
# Outside a pull (no `stale_days`) the guard never gives way.
with pytest.raises(_runner.Suspect):
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
assert _read_and_stamp(state, module, limits).ids == ["1"]
assert json.loads(state["ats:count:fake:acme"])["count"] == 1
def test_keep_size_resets_the_guards_count_unless_told_not_to() -> None:
state = {"ats:count:fake:acme": json.dumps({"asks": "[]", "count": 40})}
ats.keep_size(state, "fake:acme", 40, reset=False)
assert "ats:count:fake:acme" in state
ats.keep_size(state, "fake:acme", 3)
assert "ats:count:fake:acme" not in state
assert state["ats:size:fake:acme"] == "3"
def test_a_filter_that_maps_to_no_host_id_asks_nothing_and_is_incomplete(
monkeypatch: pytest.MonkeyPatch) -> None:
module = _listing_host(filters=Filters(country="ids"))
module.facets = lambda get, slug: {"country": {}}
module.ids = ["1", "2"]
state: dict[str, str] = {}
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
Ask(countries=("DK",)))
assert (listing.rows, listing.ids, listing.complete, listing.filtered) == (
[], [], False, True)
# Through the plugin: stamped, but not complete, so the settle closes nothing.
monkeypatch.setitem(readers.HOSTS, "fake", module)
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b'["1"]')
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
"params": {"countries": ["DK"]}}], "run": "r"})
assert list(source.fetch()) == []
assert (source.state["ats:ok:fake:acme"], source.state["ats:complete:fake:acme"]) == (
"r", "")
def test_a_filtered_read_that_cannot_ask_remote_says_so() -> None:
reach = {"countries": ["DK"], "remote": True}
blind = _listing_host(filters=Filters(country="many"))
ask = _runner.ask_for(blind, None, reach)
assert (ask.countries, ask.remote) == (("DK",), None)
assert _runner.remote_wanted(blind, None, reach)
listing = _runner.read_listing(blind, "acme", _listed(blind), ask,
remote=_runner.remote_wanted(blind, None, reach))
assert listing.filtered and listing.remote_unasked
state: dict[str, str] = {}
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
assert state["ats:remote_unasked:fake:acme"] == "1"
# A host that filters on remote asks it as a series of its own.
sees = _listing_host(filters=Filters(country="many", remote="many"))
ask = _runner.ask_for(sees, None, reach)
listing = _runner.read_listing(sees, "acme", _listed(sees), ask, remote=True)
assert listing.filtered and not listing.remote_unasked
# A params' remote of False asks nothing and wants nothing.
assert not _runner.remote_wanted(blind, {"remote": False}, reach)
+7 -17
View File
@@ -103,8 +103,6 @@ def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
both = _whole(filters=Filters(country="many", remote="many")) both = _whole(filters=Filters(country="many", remote="many"))
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True}) ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
assert (ask.countries, ask.remote) == (("DE",), True) assert (ask.countries, ask.remote) == (("DE",), True)
# A module without READS reads the whole board.
assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
def test_a_searchs_params_go_whole_into_the_ask_with_its_key( def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
@@ -139,11 +137,12 @@ def test_ids_filters_are_mapped_through_the_facets() -> None:
ask = Ask(countries=("DK", "XX")) ask = Ask(countries=("DK", "XX"))
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}}) [mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
assert mapped.countries == ("h-1",) assert mapped.countries == ("h-1",)
# None known: the host has nothing in reach, and the read is empty. # None known: nothing is asked, which is no evidence, so the read is
# incomplete and closes nothing.
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == [] assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)), listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
facets={"country": {}}) facets={"country": {}})
assert (listing.rows, listing.complete, listing.filtered) == ([], True, True) assert (listing.rows, listing.complete, listing.filtered) == ([], False, True)
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None: def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
@@ -189,17 +188,6 @@ def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
_runner.read_listing(module, "acme", web) _runner.read_listing(module, "acme", web)
def test_a_module_without_reads_gets_the_older_signatures() -> None:
module: Any = types.ModuleType("old")
module.NAME = "old"
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
listing = _runner.read_listing(module, "acme",
Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
Ask(countries=("DK",)))
assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None: def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
pages = [["1", "2"], ["3"]] pages = [["1", "2"], ["3"]]
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages), listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
@@ -333,9 +321,11 @@ def test_a_suspect_read_far_under_the_last_count_fails() -> None:
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]' web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
with pytest.raises(_runner.Suspect, match="under 10% of the 20"): with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
_runner.read(state, "ats", "t", module, "acme", web) _runner.read(state, "ats", "t", module, "acme", web)
# A reader whose empty answer is trusted reads it as it is. # A reader whose empty answer is trusted is held too, from a count of
# `SUSPECT_MIN`.
module.READS = module.READS.model_copy(update={"empty": "trusted"}) module.READS = module.READS.model_copy(update={"empty": "trusted"})
assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"] with pytest.raises(_runner.Suspect):
_runner.read(state, "ats", "t", module, "acme", web)
def test_the_guard_compares_only_reads_that_asked_the_same() -> None: def test_the_guard_compares_only_reads_that_asked_the_same() -> None: