Guard every read that is not a sweep, treat a read that asks nothing as incomplete, and mark a filtered read that could not ask remote, so no board loses its roles to a changed answer or a filter it could not send
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
4e827665dd
commit
36bc639384
4 files changed
+332
-116
No files matched your search
+41
-23
@@ -16,7 +16,9 @@ The allow-list and the classifier are ported from `openings.go` and
|
|||||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||||
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
||||||
its slug is. Every board is read independently; one that fails is logged,
|
its slug is. Every board is read independently; one that fails is logged,
|
||||||
recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
||||||
|
whose read lists under a tenth of what it listed last time (`_runner.read`'s
|
||||||
|
guard), until it has failed that way for the pull's `stale_days`. A board's
|
||||||
filter is the `reach` option the pull hands it (the agents', see
|
filter is the `reach` option the pull hands it (the agents', see
|
||||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||||
@@ -26,7 +28,8 @@ the whole board: it prunes no per-role state.
|
|||||||
A board's size (`board_size`) is set once, when the board is added
|
A board's size (`board_size`) is set once, when the board is added
|
||||||
(`app.jobs.sources.set_board`) or rechecked, and kept under
|
(`app.jobs.sources.set_board`) or rechecked, and kept under
|
||||||
`ats:size:<target>` with the day it was checked under
|
`ats:size:<target>` with the day it was checked under
|
||||||
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small
|
`ats:sizechecked:<target>` (`keep_size`, which also resets the guard's count,
|
||||||
|
so a Recheck accepts a board that shrank); the reach reads it to leave small
|
||||||
boards whole. A pull does not rewrite it. A board with no size yet is sized
|
boards whole. A pull does not rewrite it. A board with no size yet is sized
|
||||||
in its batch before it is read: its reach then says `unsized` and the size
|
in its batch before it is read: its reach then says `unsized` and the size
|
||||||
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
|
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
|
||||||
@@ -106,8 +109,9 @@ _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
|||||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||||
|
|
||||||
|
|
||||||
def _limits() -> _runner.Limits:
|
def _limits(stale_days: int | None = None) -> _runner.Limits:
|
||||||
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
|
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP,
|
||||||
|
stale_days=stale_days)
|
||||||
|
|
||||||
|
|
||||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||||
@@ -339,8 +343,7 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
|||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
module = readers.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
get = _getter(client, host, slug)
|
get = _getter(client, host, slug)
|
||||||
if (_runner.declared(module) and module.READS.listing == "paged"
|
if module.READS.listing == "paged" and module.READS.complete == "stated":
|
||||||
and module.READS.complete == "stated"):
|
|
||||||
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
|
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
|
||||||
if isinstance(total, int):
|
if isinstance(total, int):
|
||||||
return total
|
return total
|
||||||
@@ -348,11 +351,18 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
|||||||
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
||||||
|
|
||||||
|
|
||||||
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
|
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
||||||
|
reset: bool = True) -> None:
|
||||||
"""Keep board `target`'s size and the day it was checked (ISO,
|
"""Keep board `target`'s size and the day it was checked (ISO,
|
||||||
`today`, else today in UTC)."""
|
`today`, else today in UTC). With `reset`, the default and what a
|
||||||
|
Recheck does, the 10% guard's count goes too (`ats:count:<target>`), so
|
||||||
|
the next read of a board that really shrank is taken as it is instead of
|
||||||
|
failing as suspect; the plugin sizing a board as it reads it passes
|
||||||
|
False, since its count is the read it just stamped."""
|
||||||
state[f"ats:size:{target}"] = str(size)
|
state[f"ats:size:{target}"] = str(size)
|
||||||
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
|
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
|
||||||
|
if reset:
|
||||||
|
state.pop(f"ats:count:{target}", None)
|
||||||
|
|
||||||
|
|
||||||
failure_reason = _runner.failure_reason
|
failure_reason = _runner.failure_reason
|
||||||
@@ -400,14 +410,15 @@ class ATSSource(Source):
|
|||||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||||
module = readers.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
get = _getter(client, host, slug)
|
get = _getter(client, host, slug)
|
||||||
ask, unsized = self._ask(client, target, module)
|
ask, unsized, remote = self._ask(client, target, module)
|
||||||
|
stale_days = self.options.get("stale_days")
|
||||||
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
||||||
limits=_limits())
|
limits=_limits(int(stale_days) if stale_days else None),
|
||||||
|
remote=remote)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.warning("ats: %s failed (%s)", target, exc)
|
log.warning("ats: %s failed (%s)", target, exc)
|
||||||
_runner.record_failure(self.state, failing, exc)
|
_runner.record_failure(self.state, failing, exc)
|
||||||
return
|
return
|
||||||
self.state.pop(failing, None)
|
|
||||||
|
|
||||||
def compound(source_id: str) -> str:
|
def compound(source_id: str) -> str:
|
||||||
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
||||||
@@ -435,11 +446,13 @@ class ATSSource(Source):
|
|||||||
yield item_of(row, target, label)
|
yield item_of(row, target, label)
|
||||||
# Stamped only once every row went out, so a board that blew up half
|
# Stamped only once every row went out, so a board that blew up half
|
||||||
# way through is neither marked ok nor has a partial id list treated
|
# way through is neither marked ok nor has a partial id list treated
|
||||||
# as the whole of it.
|
# as the whole of it, and it stays failing until then: a row that
|
||||||
|
# fails to store (`app.jobs.pull.run_batch`) leaves the record in place.
|
||||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||||
[compound(i) for i in listing.ids])
|
[compound(i) for i in listing.ids])
|
||||||
|
self.state.pop(failing, None)
|
||||||
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
||||||
keep_size(self.state, target, len(listing.ids))
|
keep_size(self.state, target, len(listing.ids), reset=False)
|
||||||
if detail_sets is not None:
|
if detail_sets is not None:
|
||||||
# A complete listing drops the ids it no longer lists.
|
# A complete listing drops the ids it no longer lists.
|
||||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||||
@@ -447,9 +460,11 @@ class ATSSource(Source):
|
|||||||
self.state[key] = json.dumps(sorted(ids))
|
self.state[key] = json.dumps(sorted(ids))
|
||||||
|
|
||||||
def _ask(self, client: httpx.Client, target: str,
|
def _ask(self, client: httpx.Client, target: str,
|
||||||
module: Any) -> tuple[readers.Ask, bool]:
|
module: Any) -> tuple[readers.Ask, bool, bool]:
|
||||||
"""What `target` is asked for, from the reach its batch handed it and
|
"""What `target` is asked for, from the reach its batch handed it and
|
||||||
its params, and whether it still needs a size. A board with no size
|
its params, whether it still needs a size, and whether that reach
|
||||||
|
and params want remote roles (`_runner.remote_wanted`, which the
|
||||||
|
runner compares with what it could ask). A board with no size
|
||||||
yet (`unsized` in its reach) whose reach would narrow the read is
|
yet (`unsized` in its reach) whose reach would narrow the read is
|
||||||
sized first (`board_size`, kept by `keep_size`), and read whole under
|
sized first (`board_size`, kept by `keep_size`), and read whole under
|
||||||
`small` roles. One whose read is whole anyway is sized by that read,
|
`small` roles. One whose read is whole anyway is sized by that read,
|
||||||
@@ -457,16 +472,18 @@ class ATSSource(Source):
|
|||||||
given = self.options.get("reach")
|
given = self.options.get("reach")
|
||||||
params, keys = self.params_for(target), self.options.get("keys")
|
params, keys = self.params_for(target), self.options.get("keys")
|
||||||
if not isinstance(given, dict) or not given.get("unsized"):
|
if not isinstance(given, dict) or not given.get("unsized"):
|
||||||
return _runner.ask_for(module, params, given, keys=keys), False
|
return (_runner.ask_for(module, params, given, keys=keys), False,
|
||||||
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
|
_runner.remote_wanted(module, params, given))
|
||||||
ask = _runner.ask_for(module, params, reach or None, keys=keys)
|
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None
|
||||||
|
ask = _runner.ask_for(module, params, reach, keys=keys)
|
||||||
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
|
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
|
||||||
return ask, True
|
return ask, True, _runner.remote_wanted(module, params, reach)
|
||||||
size = board_size(target, client=client)
|
size = board_size(target, client=client)
|
||||||
keep_size(self.state, target, size)
|
keep_size(self.state, target, size, reset=False)
|
||||||
if size < int(given.get("small") or 0):
|
if size < int(given.get("small") or 0):
|
||||||
|
reach = None
|
||||||
ask = _runner.ask_for(module, params, None, keys=keys)
|
ask = _runner.ask_for(module, params, None, keys=keys)
|
||||||
return ask, False
|
return ask, False, _runner.remote_wanted(module, params, reach)
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------ the per-role reads
|
# ------------------------------------------------------ the per-role reads
|
||||||
@@ -567,7 +584,8 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
|
|||||||
|
|
||||||
def item_of(row: Row, target: str, label: str) -> Item:
|
def item_of(row: Row, target: str, label: str) -> Item:
|
||||||
"""One row of board `target` as the `Item` the store takes: its id
|
"""One row of board `target` as the `Item` the store takes: its id
|
||||||
compound, `host:slug:id`, as `roles.source_id` has it."""
|
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
|
||||||
|
date is dated now, as `Item` itself would date it."""
|
||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
description = row.get("description") or ""
|
description = row.get("description") or ""
|
||||||
return Item(
|
return Item(
|
||||||
@@ -576,7 +594,7 @@ def item_of(row: Row, target: str, label: str) -> Item:
|
|||||||
title=row["title"],
|
title=row["title"],
|
||||||
excerpt=description[:300],
|
excerpt=description[:300],
|
||||||
body=description,
|
body=description,
|
||||||
published_at=row.get("posted"),
|
published_at=row.get("posted") or datetime.now(UTC),
|
||||||
subscription=label,
|
subscription=label,
|
||||||
kind="role",
|
kind="role",
|
||||||
raw=_runner.raw_of(row) | {
|
raw=_runner.raw_of(row) | {
|
||||||
|
|||||||
+152
-74
@@ -5,23 +5,25 @@ A reader says how its host answers; this module reads it: the guarded fetch
|
|||||||
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
||||||
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
||||||
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
||||||
the sweep of an oversized paged board across pulls, the 10% guard on a
|
the sweep of an oversized paged board across pulls, the 10% guard on every
|
||||||
filtered or `empty="suspect"` read (`read`), the per-role detail reads
|
read that is not a sweep (`read`), the per-role detail reads
|
||||||
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||||
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
||||||
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
||||||
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
||||||
as they were.
|
as they were. Every reader declares `READS`; a module without it is refused
|
||||||
|
by the package before it gets here.
|
||||||
A module without `READS` is read with the older signatures, `feed_url(slug)`
|
|
||||||
and `page(get, slug, cursor)`, and never gets an `Ask`.
|
|
||||||
|
|
||||||
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
||||||
|
|
||||||
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
|
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
|
||||||
"seen", "rows", "whole"}`, continued by the next pull
|
"seen", "rows", "whole"}`, continued by the next pull
|
||||||
count `{"asks", "count"}`, the roles the last whole read listed, for
|
count `{"asks", "count"}`, the roles the last read that was not a
|
||||||
the 10% guard
|
sweep listed, for the 10% guard
|
||||||
|
failing `{"since", "reason"}`, kept by the Source (`record_failure`);
|
||||||
|
the guard gives way once `since` is `Limits.stale_days` old
|
||||||
|
remote_unasked "1" when the reach wanted remote and the read could not
|
||||||
|
ask for it (`Listing.remote_unasked`), else ""
|
||||||
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
||||||
swept "1" when the last stamped listing finished a sweep carried over
|
swept "1" when the last stamped listing finished a sweep carried over
|
||||||
from an earlier pull (`Listing.swept`), else "". Offset paging
|
from an earlier pull (`Listing.swept`), else "". Offset paging
|
||||||
@@ -87,10 +89,16 @@ DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "count
|
|||||||
#: How long a board's `facets` are kept before they are asked again.
|
#: How long a board's `facets` are kept before they are asked again.
|
||||||
FACETS_TTL = timedelta(days=1)
|
FACETS_TTL = timedelta(days=1)
|
||||||
|
|
||||||
#: A filtered or `empty="suspect"` read listing under this share of what the
|
#: A read listing under this share of what the same read listed last time
|
||||||
#: same read listed last time counts as a failure.
|
#: counts as a failure (`read`).
|
||||||
SUSPECT_SHARE = 0.1
|
SUSPECT_SHARE = 0.1
|
||||||
|
|
||||||
|
#: The last count from which the guard holds every read. A read that is
|
||||||
|
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
||||||
|
#: a count of 1, as it was before every read was; a whole read of a board
|
||||||
|
#: that listed a handful of roles can drop to none by itself.
|
||||||
|
SUSPECT_MIN = 10
|
||||||
|
|
||||||
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
||||||
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
|
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
|
||||||
|
|
||||||
@@ -105,8 +113,8 @@ class TooLarge(ValueError):
|
|||||||
|
|
||||||
|
|
||||||
class Suspect(Exception):
|
class Suspect(Exception):
|
||||||
"""A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE`
|
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
|
||||||
of what it listed last time."""
|
(see `read`)."""
|
||||||
|
|
||||||
|
|
||||||
class MissingKey(ValueError):
|
class MissingKey(ValueError):
|
||||||
@@ -115,12 +123,15 @@ class MissingKey(ValueError):
|
|||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class Limits:
|
class Limits:
|
||||||
"""What one Source allows a read: pages, the pause between requests, and
|
"""What one Source allows a read: pages, the pause between requests, the
|
||||||
the detail reads a pull."""
|
detail reads a pull, and the days after which the 10% guard accepts the
|
||||||
|
answer it keeps refusing (`stale_days`, the pull's `STALE_DAYS`; None,
|
||||||
|
for a Source run outside a pull, never)."""
|
||||||
|
|
||||||
max_pages: int = MAX_PAGES
|
max_pages: int = MAX_PAGES
|
||||||
page_delay: float = PAGE_DELAY
|
page_delay: float = PAGE_DELAY
|
||||||
detail_cap: int = DETAIL_CAP
|
detail_cap: int = DETAIL_CAP
|
||||||
|
stale_days: int | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
@@ -134,7 +145,9 @@ class Listing:
|
|||||||
neither are the whole board. `swept` says a complete listing finished a
|
neither are the whole board. `swept` says a complete listing finished a
|
||||||
sweep carried over from an earlier pull: `ids` were read across pulls,
|
sweep carried over from an earlier pull: `ids` were read across pulls,
|
||||||
and roles closing meanwhile may have shifted an offset-paged host's later
|
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||||
roles back past the cursor, so `ids` can miss a role that is still open."""
|
roles back past the cursor, so `ids` can miss a role that is still open.
|
||||||
|
`remote_unasked` says a filtered read was meant to ask for remote roles
|
||||||
|
as well and could not, so its `ids` say nothing of the remote roles."""
|
||||||
|
|
||||||
rows: list[Row]
|
rows: list[Row]
|
||||||
complete: bool
|
complete: bool
|
||||||
@@ -144,12 +157,14 @@ class Listing:
|
|||||||
asks: str = ""
|
asks: str = ""
|
||||||
narrowed: bool = False
|
narrowed: bool = False
|
||||||
swept: bool = False
|
swept: bool = False
|
||||||
|
remote_unasked: bool = False
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- modules
|
# --------------------------------------------------------------- modules
|
||||||
|
|
||||||
def declared(module: ModuleType) -> bool:
|
def declared(module: ModuleType) -> bool:
|
||||||
"""Whether `module` declares `READS`, and so takes `ask`."""
|
"""Whether `module` declares `READS`. Every reader does, so the runner no
|
||||||
|
longer asks; `app.jobs.reach` still does."""
|
||||||
return isinstance(getattr(module, "READS", None), Reads)
|
return isinstance(getattr(module, "READS", None), Reads)
|
||||||
|
|
||||||
|
|
||||||
@@ -334,6 +349,33 @@ def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
|
|||||||
return count
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def _wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None,
|
||||||
|
reach: Mapping[str, Any] | None) -> tuple[dict[str, Any], bool]:
|
||||||
|
"""The dimensions `reach` and then the params set, params winning, before
|
||||||
|
they are narrowed to what the reader declares, and whether the read is
|
||||||
|
wide. A reach of more than `MAX_SERIES` countries on a `one` country
|
||||||
|
filter is dropped whole, which makes the read wide; a `remote` of False
|
||||||
|
asks nothing."""
|
||||||
|
given = reach or {}
|
||||||
|
reached = _dimensions(module, given)
|
||||||
|
wide = bool(given.get("wide"))
|
||||||
|
if module.READS.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
||||||
|
reached = {}
|
||||||
|
wide = True
|
||||||
|
wanted = reached | _dimensions(module, dict(subscription_params or {}))
|
||||||
|
if wanted.get("remote") is False:
|
||||||
|
del wanted["remote"]
|
||||||
|
return wanted, wide
|
||||||
|
|
||||||
|
|
||||||
|
def remote_wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||||
|
reach: Mapping[str, Any] | None = None) -> bool:
|
||||||
|
"""Whether the reach or the params want remote roles, whether or not the
|
||||||
|
reader can ask for them. `read_listing` compares it with what the series
|
||||||
|
asked (`Listing.remote_unasked`)."""
|
||||||
|
return _wanted(module, subscription_params, reach)[0].get("remote") is True
|
||||||
|
|
||||||
|
|
||||||
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||||
reach: Mapping[str, Any] | None = None, *,
|
reach: Mapping[str, Any] | None = None, *,
|
||||||
keys: Mapping[str, Any] | None = None) -> Ask:
|
keys: Mapping[str, Any] | None = None) -> Ask:
|
||||||
@@ -346,7 +388,8 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
|
|||||||
dropped. A `remote` of False asks nothing. A search's params go whole into
|
dropped. A `remote` of False asks nothing. A search's params go whole into
|
||||||
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
||||||
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
||||||
A module without `READS` gets `Ask()`.
|
A remote the reader cannot filter on is dropped here; `remote_wanted`
|
||||||
|
still says it was wanted.
|
||||||
|
|
||||||
Fetch terms (`reach["terms"]`) go only to a board reader that declares
|
Fetch terms (`reach["terms"]`) go only to a board reader that declares
|
||||||
`query="one"`, and only on a wide read: `reach["wide"]` (a company set
|
`query="one"`, and only on a wide read: `reach["wide"]` (a company set
|
||||||
@@ -358,19 +401,10 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
|
|||||||
whole, since remote without its countries would read only the remote
|
whole, since remote without its countries would read only the remote
|
||||||
roles; over it with terms alone, the terms go too and the board is read
|
roles; over it with terms alone, the terms go too and the board is read
|
||||||
unfiltered."""
|
unfiltered."""
|
||||||
if not declared(module):
|
|
||||||
return Ask()
|
|
||||||
reads: Reads = module.READS
|
reads: Reads = module.READS
|
||||||
params = dict(subscription_params or {})
|
params = dict(subscription_params or {})
|
||||||
given = reach or {}
|
given = reach or {}
|
||||||
reached = _dimensions(module, given)
|
wanted, wide = _wanted(module, params, given)
|
||||||
wide = bool(given.get("wide"))
|
|
||||||
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
|
||||||
reached = {}
|
|
||||||
wide = True
|
|
||||||
wanted = reached | _dimensions(module, params)
|
|
||||||
if wanted.get("remote") is False:
|
|
||||||
del wanted["remote"]
|
|
||||||
declared_now = reads.filters.declared()
|
declared_now = reads.filters.declared()
|
||||||
wanted = {name: value for name, value in wanted.items() if name in declared_now}
|
wanted = {name: value for name, value in wanted.items() if name in declared_now}
|
||||||
terms = _terms(given.get("terms"))
|
terms = _terms(given.get("terms"))
|
||||||
@@ -400,7 +434,7 @@ def narrowed(module: ModuleType, ask: Ask) -> bool:
|
|||||||
query. Such a read is never the whole board, nor the board within its
|
query. Such a read is never the whole board, nor the board within its
|
||||||
places, so it closes nothing. A search's query is what
|
places, so it closes nothing. A search's query is what
|
||||||
the search is, and does not count."""
|
the search is, and does not count."""
|
||||||
board = not declared(module) or module.READS.scope == "board"
|
board = module.READS.scope == "board"
|
||||||
return bool(ask.terms) or (board and bool(ask.query))
|
return bool(ask.terms) or (board and bool(ask.query))
|
||||||
|
|
||||||
|
|
||||||
@@ -417,10 +451,7 @@ def series(module: ModuleType, ask: Ask,
|
|||||||
remote part beside places is then left out. Each of those is then asked
|
remote part beside places is then left out. Each of those is then asked
|
||||||
once per fetch term (`ask.terms`), with the term as `query`: the host
|
once per fetch term (`ask.terms`), with the term as `query`: the host
|
||||||
intersects a keyword with its facets, as the survey found on every
|
intersects a keyword with its facets, as the survey found on every
|
||||||
host, so the union by id is "one of the terms, in these places". `[ask]`
|
host, so the union by id is "one of the terms, in these places"."""
|
||||||
for a module without `READS`."""
|
|
||||||
if not declared(module):
|
|
||||||
return [ask]
|
|
||||||
if ask.remote and (ask.countries or ask.places):
|
if ask.remote and (ask.countries or ask.places):
|
||||||
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
||||||
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
||||||
@@ -450,8 +481,8 @@ def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]
|
|||||||
known = known_facets.get(dimension) or {}
|
known = known_facets.get(dimension) or {}
|
||||||
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
|
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
|
||||||
if not mapped:
|
if not mapped:
|
||||||
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
|
log.warning("readers: %s knows none of %s %s; nothing asked", module.NAME,
|
||||||
", ".join(values))
|
dimension, ", ".join(values))
|
||||||
return []
|
return []
|
||||||
if dimension == "country":
|
if dimension == "country":
|
||||||
countries = mapped
|
countries = mapped
|
||||||
@@ -497,7 +528,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
|||||||
for index, ask in enumerate(asks):
|
for index, ask in enumerate(asks):
|
||||||
if index:
|
if index:
|
||||||
time.sleep(limits.page_delay)
|
time.sleep(limits.page_delay)
|
||||||
url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug)
|
url = module.feed_url(slug, ask)
|
||||||
try:
|
try:
|
||||||
body = get(url)
|
body = get(url)
|
||||||
except TooLarge:
|
except TooLarge:
|
||||||
@@ -514,7 +545,8 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
|||||||
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
|
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
|
||||||
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
|
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
|
||||||
can_sweep: bool = False,
|
can_sweep: bool = False,
|
||||||
facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing:
|
facets: Mapping[str, Mapping[str, str]] | None = None,
|
||||||
|
remote: bool = False) -> Listing:
|
||||||
"""One target's listing, read and parsed. Raises on any failure of the
|
"""One target's listing, read and parsed. Raises on any failure of the
|
||||||
first request, and on a reader that does not understand an answer.
|
first request, and on a reader that does not understand an answer.
|
||||||
|
|
||||||
@@ -532,22 +564,35 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
its series and cursor; the sweep is complete when it reaches the end,
|
its series and cursor; the sweep is complete when it reaches the end,
|
||||||
and `ids` then hold every id it listed, and the listing is `swept`. A
|
and `ids` then hold every id it listed, and the listing is `swept`. A
|
||||||
read at the reader's `ceiling` is not swept: the host answers nothing
|
read at the reader's `ceiling` is not swept: the host answers nothing
|
||||||
past it."""
|
past it.
|
||||||
|
|
||||||
|
A filtered read that asks no series, because an `ids` dimension names
|
||||||
|
no value the host's facets know, asked nothing, which is no evidence of
|
||||||
|
anything: it is incomplete and closes nothing. `filtered` is a board's
|
||||||
|
only: a search's places are what the search is, as its query is.
|
||||||
|
`remote` says the reach or the params wanted remote roles; when no series
|
||||||
|
asked for them (a reader without a remote filter, or one whose facets
|
||||||
|
state none beside places) the listing is `remote_unasked`."""
|
||||||
limits = limits or Limits()
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
|
reads: Reads = module.READS
|
||||||
asks = series(module, ask, facets)
|
asks = series(module, ask, facets)
|
||||||
is_filtered = filtered(ask)
|
is_filtered = filtered(ask) and reads.scope == "board"
|
||||||
by_words = narrowed(module, ask)
|
by_words = narrowed(module, ask)
|
||||||
|
unasked = (is_filtered and (remote or bool(ask.remote))
|
||||||
|
and not any(a.remote or a.remote_ids for a in asks))
|
||||||
key = fingerprint(asks)
|
key = fingerprint(asks)
|
||||||
if not asks:
|
if not asks:
|
||||||
return Listing([], True, [], is_filtered, None, key, by_words)
|
log.warning("readers: %s:%s: the filter maps to nothing the host knows; "
|
||||||
|
"nothing asked, not closing", module.NAME, slug)
|
||||||
|
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
||||||
if not hasattr(module, "page"):
|
if not hasattr(module, "page"):
|
||||||
rows = _whole(module, slug, get, asks, limits)
|
whole_rows = _whole(module, slug, get, asks, limits)
|
||||||
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words)
|
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
||||||
|
key, by_words, remote_unasked=unasked)
|
||||||
|
|
||||||
reads: Reads | None = module.READS if declared(module) else None
|
never = reads.complete == "never"
|
||||||
never = reads is not None and reads.complete == "never"
|
ceiling = reads.ceiling
|
||||||
ceiling = reads.ceiling if reads is not None else None
|
|
||||||
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
||||||
resumed = bool(sweep and sweep.get("asks") == key)
|
resumed = bool(sweep and sweep.get("asks") == key)
|
||||||
if sweep and resumed:
|
if sweep and resumed:
|
||||||
@@ -568,10 +613,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
if pages:
|
if pages:
|
||||||
time.sleep(limits.page_delay)
|
time.sleep(limits.page_delay)
|
||||||
try:
|
try:
|
||||||
if reads is not None:
|
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
||||||
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
|
||||||
else:
|
|
||||||
data, results, total, cursor = module.page(get, slug, cursor)
|
|
||||||
except httpx.HTTPStatusError as exc:
|
except httpx.HTTPStatusError as exc:
|
||||||
if not pages or exc.response.status_code != 429:
|
if not pages or exc.response.status_code != 429:
|
||||||
raise
|
raise
|
||||||
@@ -603,15 +645,17 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
listed = list(dict.fromkeys(earlier + list(rows)))
|
listed = list(dict.fromkeys(earlier + list(rows)))
|
||||||
if not stop:
|
if not stop:
|
||||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
|
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
|
||||||
resumed and whole)
|
resumed and whole, unasked)
|
||||||
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
||||||
module.NAME, slug, pages, stop)
|
module.NAME, slug, pages, stop)
|
||||||
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
||||||
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
||||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words)
|
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words,
|
||||||
|
remote_unasked=unasked)
|
||||||
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
||||||
"rows": listed, "whole": whole}
|
"rows": listed, "whole": whole}
|
||||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words)
|
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words,
|
||||||
|
remote_unasked=unasked)
|
||||||
|
|
||||||
|
|
||||||
def _storable(cursor: Any) -> bool:
|
def _storable(cursor: Any) -> bool:
|
||||||
@@ -637,7 +681,7 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
|||||||
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
|
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
|
||||||
(a country, a place, or remote),
|
(a country, a place, or remote),
|
||||||
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
|
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
|
||||||
if not declared(module) or not hasattr(module, "facets"):
|
if not hasattr(module, "facets"):
|
||||||
return {}
|
return {}
|
||||||
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
|
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
|
||||||
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
|
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
|
||||||
@@ -659,46 +703,79 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
|||||||
|
|
||||||
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
|
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
|
||||||
slug: str, get: Get, ask: Ask | None = None, *,
|
slug: str, get: Get, ask: Ask | None = None, *,
|
||||||
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
|
limits: Limits | None = None, can_sweep: bool = True, remote: bool = False) -> Listing:
|
||||||
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
||||||
the facets it needs, the sweep it continues, and the 10% guard. A read
|
the facets it needs, the sweep it continues, and the 10% guard. Nothing
|
||||||
that is filtered, narrowed by terms, or whose reader declares
|
is written here but the facets; `stamp` keeps the rest once the rows are
|
||||||
`empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same
|
stored.
|
||||||
read listed last time (the same `fingerprint`, so a term read is only
|
|
||||||
held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
The guard holds every read that is not a sweep: one that listed under
|
||||||
the rest once the rows are stored."""
|
`SUSPECT_SHARE` of what the same read listed last time (the same
|
||||||
|
`fingerprint`, so a term read is only held to a term read with the same
|
||||||
|
terms and places) raises `Suspect`, and the Source records the target
|
||||||
|
as failing, so its roles stay open. A host whose answer changed shape
|
||||||
|
parses to no rows with a 200, and closing a board on that would lose it
|
||||||
|
whole. It holds from a last count of `SUSPECT_MIN`, or of 1 for a read
|
||||||
|
that is filtered, narrowed or of an `empty="suspect"` reader. A board
|
||||||
|
that really shrank is accepted once the target has been failing for
|
||||||
|
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
||||||
|
stamp then resets the count; a Recheck resets it at once
|
||||||
|
(`plugins.ats.keep_size`)."""
|
||||||
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||||
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
||||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets)
|
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
||||||
suspect = declared(module) and module.READS.empty == "suspect"
|
remote=remote)
|
||||||
|
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
||||||
last = _load(state, f"{prefix}:count:{target}")
|
last = _load(state, f"{prefix}:count:{target}")
|
||||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||||
if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count
|
if (listing.sweep is None and count >= (1 if held else SUSPECT_MIN)
|
||||||
and len(listing.ids) < SUSPECT_SHARE * count):
|
and len(listing.ids) < SUSPECT_SHARE * count):
|
||||||
raise Suspect(f"listed {len(listing.ids)} roles, under "
|
why = (f"listed {len(listing.ids)} roles, under "
|
||||||
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
||||||
|
if _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
|
||||||
|
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
|
||||||
|
module.NAME, slug, why, limits.stale_days)
|
||||||
|
return listing
|
||||||
|
raise Suspect(why)
|
||||||
return listing
|
return listing
|
||||||
|
|
||||||
|
|
||||||
|
def _failing_for(state: Mapping[str, str], key: str, days: int | None) -> bool:
|
||||||
|
"""Whether the failure kept under `key` began at least `days` ago; False
|
||||||
|
for `days` None or no record."""
|
||||||
|
since = str(_load(state, key).get("since") or "")
|
||||||
|
if days is None or not since:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
began = datetime.fromisoformat(since)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
return datetime.now(UTC) - began >= timedelta(days=days)
|
||||||
|
|
||||||
|
|
||||||
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||||
listing: Listing, ids: list[str]) -> None:
|
listing: Listing, ids: list[str]) -> None:
|
||||||
"""Record a read whose rows are stored: `ok` (the run's marker),
|
"""Record a read whose rows are stored: `ok` (the run's marker),
|
||||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||||
against), whether it finished a sweep from earlier pulls (`swept`), the
|
against), whether it finished a sweep from earlier pulls (`swept`),
|
||||||
|
whether a filtered read left remote unasked (`remote_unasked`), the
|
||||||
sweep to continue or none, and the count for the 10% guard once a read
|
sweep to continue or none, and the count for the 10% guard once a read
|
||||||
is whole."""
|
is not a sweep. A read that asked nothing keeps the count it had."""
|
||||||
state[f"{prefix}:ok:{target}"] = run
|
state[f"{prefix}:ok:{target}"] = run
|
||||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||||
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||||
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
||||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||||
|
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
||||||
if listing.sweep is not None:
|
if listing.sweep is not None:
|
||||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||||
else:
|
return
|
||||||
state.pop(f"{prefix}:sweep:{target}", None)
|
state.pop(f"{prefix}:sweep:{target}", None)
|
||||||
|
if listing.asks != fingerprint([]):
|
||||||
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
|
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
|
||||||
"count": len(listing.ids)})
|
"count": len(listing.ids)})
|
||||||
|
|
||||||
@@ -766,7 +843,7 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
|||||||
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
||||||
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
||||||
name, slug, row["source_id"], ", ".join(ignored))
|
name, slug, row["source_id"], ", ".join(ignored))
|
||||||
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment]
|
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
|
||||||
if taken.get("location") and "places" not in taken:
|
if taken.get("location") and "places" not in taken:
|
||||||
taken["places"] = [{"text": taken["location"], "remote": None}]
|
taken["places"] = [{"text": taken["location"], "remote": None}]
|
||||||
filled[key] = taken
|
filled[key] = taken
|
||||||
@@ -776,12 +853,12 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
|||||||
for row in rows:
|
for row in rows:
|
||||||
key = compound(row)
|
key = compound(row)
|
||||||
if key in filled:
|
if key in filled:
|
||||||
out.append(row | {"description": ""} | filled[key]) # type: ignore[operator]
|
out.append(row | {"description": ""} | filled[key])
|
||||||
elif key in detailed:
|
elif key in detailed:
|
||||||
out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator]
|
out.append(row | {"description": "", "location": "", "places": [],
|
||||||
"remote": None})
|
"remote": None})
|
||||||
else:
|
else:
|
||||||
out.append(row | {"description": ""}) # type: ignore[operator]
|
out.append(row | {"description": ""})
|
||||||
if listed is not None:
|
if listed is not None:
|
||||||
done &= listed
|
done &= listed
|
||||||
failing &= listed
|
failing &= listed
|
||||||
@@ -791,8 +868,9 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
|||||||
# -------------------------------------------------------------------- raw
|
# -------------------------------------------------------------------- raw
|
||||||
|
|
||||||
def places_of(row: Mapping[str, Any]) -> list[Place]:
|
def places_of(row: Mapping[str, Any]) -> list[Place]:
|
||||||
"""A row's places: its `places`, or, for a row without (an undeclared
|
"""A row's places: its `places`, or, for a row without (a raw row stored
|
||||||
reader's), its `location` as the one place, split nothing."""
|
before readers returned places), its `location` as the one place, split
|
||||||
|
nothing."""
|
||||||
places = row.get("places")
|
places = row.get("places")
|
||||||
if not isinstance(places, list):
|
if not isinstance(places, list):
|
||||||
location = str(row.get("location") or "").strip()
|
location = str(row.get("location") or "").strip()
|
||||||
|
|||||||
+132
-2
@@ -26,7 +26,7 @@ if str(ROOT) not in sys.path:
|
|||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import _runner # noqa: E402
|
from plugins.readers import _runner # noqa: E402
|
||||||
from plugins.readers import lever # noqa: E402
|
from plugins.readers import lever # noqa: E402
|
||||||
from plugins.readers._contract import Ask # noqa: E402
|
from plugins.readers._contract import Ask, Filters, Reads # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -253,7 +253,9 @@ def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
|
|||||||
module.slug_ok = lambda slug: slug.isalnum()
|
module.slug_ok = lambda slug: slug.isalnum()
|
||||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
module.READS = Reads(scope="board", listing="whole", complete="whole", places="one",
|
||||||
|
text="detail" if detail is not None else "listing", robots="test")
|
||||||
|
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
|
||||||
module.parse = lambda body, slug: [
|
module.parse = lambda body, slug: [
|
||||||
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
||||||
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
||||||
@@ -590,3 +592,131 @@ def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
|
|||||||
|
|
||||||
with _client(handler) as client, pytest.raises(ats._Refused):
|
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||||
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------- the guard, asks, remote unasked
|
||||||
|
|
||||||
|
def _listing_host(**reads: Any) -> Any:
|
||||||
|
"""A declared whole-listing board `fake` whose feed lists `module.ids`,
|
||||||
|
whatever is asked; `reads` override its declaration."""
|
||||||
|
module: Any = types.ModuleType("fake")
|
||||||
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||||
|
module.slug_ok = lambda slug: slug.isalnum()
|
||||||
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||||
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||||
|
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
|
||||||
|
"complete": "whole", "places": "one", "robots": "test"} | reads))
|
||||||
|
module.ids = []
|
||||||
|
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
|
||||||
|
module.parse = lambda body, slug: [
|
||||||
|
{"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
|
||||||
|
"places": [{"text": "Berlin", "remote": None}]} for i in json.loads(body)]
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def _listed(module: Any) -> Callable[..., bytes]:
|
||||||
|
return lambda url, **kwargs: json.dumps(module.ids).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _read_and_stamp(state: dict[str, str], module: Any,
|
||||||
|
limits: _runner.Limits | None = None) -> _runner.Listing:
|
||||||
|
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
|
||||||
|
limits=limits)
|
||||||
|
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||||
|
return listing
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unfiltered_whole_read_of_nothing_after_forty_is_suspect() -> None:
|
||||||
|
"""A Greenhouse-style board, trusted, unfiltered: a changed answer that
|
||||||
|
parses to no rows is refused rather than read as a board that closed."""
|
||||||
|
module = _listing_host()
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
module.ids = [str(n) for n in range(40)]
|
||||||
|
_read_and_stamp(state, module)
|
||||||
|
module.ids = []
|
||||||
|
with pytest.raises(_runner.Suspect, match="listed 0 roles, under 10% of the 40"):
|
||||||
|
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_read_under_suspect_min_is_not_held() -> None:
|
||||||
|
module = _listing_host()
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
module.ids = [str(n) for n in range(5)]
|
||||||
|
_read_and_stamp(state, module)
|
||||||
|
module.ids = []
|
||||||
|
assert _read_and_stamp(state, module).ids == []
|
||||||
|
assert json.loads(state["ats:count:fake:acme"])["count"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_same_suspect_answer_is_accepted_after_stale_days() -> None:
|
||||||
|
module = _listing_host()
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
module.ids = [str(n) for n in range(40)]
|
||||||
|
_read_and_stamp(state, module)
|
||||||
|
module.ids = ["1"]
|
||||||
|
failing = "ats:failing:fake:acme"
|
||||||
|
recent = (datetime.now(UTC) - timedelta(days=20)).isoformat(timespec="seconds")
|
||||||
|
state[failing] = json.dumps({"since": recent, "reason": "Suspect: listed 1 roles"})
|
||||||
|
limits = _runner.Limits(page_delay=0, stale_days=21)
|
||||||
|
with pytest.raises(_runner.Suspect):
|
||||||
|
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits)
|
||||||
|
|
||||||
|
old = (datetime.now(UTC) - timedelta(days=22)).isoformat(timespec="seconds")
|
||||||
|
state[failing] = json.dumps({"since": old, "reason": "Suspect: listed 1 roles"})
|
||||||
|
# Outside a pull (no `stale_days`) the guard never gives way.
|
||||||
|
with pytest.raises(_runner.Suspect):
|
||||||
|
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
|
||||||
|
assert _read_and_stamp(state, module, limits).ids == ["1"]
|
||||||
|
assert json.loads(state["ats:count:fake:acme"])["count"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_keep_size_resets_the_guards_count_unless_told_not_to() -> None:
|
||||||
|
state = {"ats:count:fake:acme": json.dumps({"asks": "[]", "count": 40})}
|
||||||
|
ats.keep_size(state, "fake:acme", 40, reset=False)
|
||||||
|
assert "ats:count:fake:acme" in state
|
||||||
|
ats.keep_size(state, "fake:acme", 3)
|
||||||
|
assert "ats:count:fake:acme" not in state
|
||||||
|
assert state["ats:size:fake:acme"] == "3"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filter_that_maps_to_no_host_id_asks_nothing_and_is_incomplete(
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
module = _listing_host(filters=Filters(country="ids"))
|
||||||
|
module.facets = lambda get, slug: {"country": {}}
|
||||||
|
module.ids = ["1", "2"]
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
|
||||||
|
Ask(countries=("DK",)))
|
||||||
|
assert (listing.rows, listing.ids, listing.complete, listing.filtered) == (
|
||||||
|
[], [], False, True)
|
||||||
|
|
||||||
|
# Through the plugin: stamped, but not complete, so the settle closes nothing.
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||||
|
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b'["1"]')
|
||||||
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
|
||||||
|
"params": {"countries": ["DK"]}}], "run": "r"})
|
||||||
|
assert list(source.fetch()) == []
|
||||||
|
assert (source.state["ats:ok:fake:acme"], source.state["ats:complete:fake:acme"]) == (
|
||||||
|
"r", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filtered_read_that_cannot_ask_remote_says_so() -> None:
|
||||||
|
reach = {"countries": ["DK"], "remote": True}
|
||||||
|
blind = _listing_host(filters=Filters(country="many"))
|
||||||
|
ask = _runner.ask_for(blind, None, reach)
|
||||||
|
assert (ask.countries, ask.remote) == (("DK",), None)
|
||||||
|
assert _runner.remote_wanted(blind, None, reach)
|
||||||
|
listing = _runner.read_listing(blind, "acme", _listed(blind), ask,
|
||||||
|
remote=_runner.remote_wanted(blind, None, reach))
|
||||||
|
assert listing.filtered and listing.remote_unasked
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||||
|
assert state["ats:remote_unasked:fake:acme"] == "1"
|
||||||
|
|
||||||
|
# A host that filters on remote asks it as a series of its own.
|
||||||
|
sees = _listing_host(filters=Filters(country="many", remote="many"))
|
||||||
|
ask = _runner.ask_for(sees, None, reach)
|
||||||
|
listing = _runner.read_listing(sees, "acme", _listed(sees), ask, remote=True)
|
||||||
|
assert listing.filtered and not listing.remote_unasked
|
||||||
|
# A params' remote of False asks nothing and wants nothing.
|
||||||
|
assert not _runner.remote_wanted(blind, {"remote": False}, reach)
|
||||||
+7
-17
@@ -103,8 +103,6 @@ def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
|
|||||||
both = _whole(filters=Filters(country="many", remote="many"))
|
both = _whole(filters=Filters(country="many", remote="many"))
|
||||||
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
|
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
|
||||||
assert (ask.countries, ask.remote) == (("DE",), True)
|
assert (ask.countries, ask.remote) == (("DE",), True)
|
||||||
# A module without READS reads the whole board.
|
|
||||||
assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
|
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
|
||||||
@@ -139,11 +137,12 @@ def test_ids_filters_are_mapped_through_the_facets() -> None:
|
|||||||
ask = Ask(countries=("DK", "XX"))
|
ask = Ask(countries=("DK", "XX"))
|
||||||
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
|
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
|
||||||
assert mapped.countries == ("h-1",)
|
assert mapped.countries == ("h-1",)
|
||||||
# None known: the host has nothing in reach, and the read is empty.
|
# None known: nothing is asked, which is no evidence, so the read is
|
||||||
|
# incomplete and closes nothing.
|
||||||
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
|
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
|
||||||
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
|
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
|
||||||
facets={"country": {}})
|
facets={"country": {}})
|
||||||
assert (listing.rows, listing.complete, listing.filtered) == ([], True, True)
|
assert (listing.rows, listing.complete, listing.filtered) == ([], False, True)
|
||||||
|
|
||||||
|
|
||||||
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
|
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
|
||||||
@@ -189,17 +188,6 @@ def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
|
|||||||
_runner.read_listing(module, "acme", web)
|
_runner.read_listing(module, "acme", web)
|
||||||
|
|
||||||
|
|
||||||
def test_a_module_without_reads_gets_the_older_signatures() -> None:
|
|
||||||
module: Any = types.ModuleType("old")
|
|
||||||
module.NAME = "old"
|
|
||||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
|
||||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
|
|
||||||
listing = _runner.read_listing(module, "acme",
|
|
||||||
Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
|
|
||||||
Ask(countries=("DK",)))
|
|
||||||
assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
|
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
|
||||||
pages = [["1", "2"], ["3"]]
|
pages = [["1", "2"], ["3"]]
|
||||||
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
|
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
|
||||||
@@ -333,9 +321,11 @@ def test_a_suspect_read_far_under_the_last_count_fails() -> None:
|
|||||||
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
|
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
|
||||||
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
|
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
|
||||||
_runner.read(state, "ats", "t", module, "acme", web)
|
_runner.read(state, "ats", "t", module, "acme", web)
|
||||||
# A reader whose empty answer is trusted reads it as it is.
|
# A reader whose empty answer is trusted is held too, from a count of
|
||||||
|
# `SUSPECT_MIN`.
|
||||||
module.READS = module.READS.model_copy(update={"empty": "trusted"})
|
module.READS = module.READS.model_copy(update={"empty": "trusted"})
|
||||||
assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"]
|
with pytest.raises(_runner.Suspect):
|
||||||
|
_runner.read(state, "ats", "t", module, "acme", web)
|
||||||
|
|
||||||
|
|
||||||
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
||||||
|
|||||||
Reference in new issue
Block a user