Guard every read that is not a sweep, treat a read that asks nothing as incomplete, and mark a filtered read that could not ask remote, so no board loses its roles to a changed answer or a filter it could not send
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
4e827665dd
commit
36bc639384
4 files changed
+332
-116
No files matched your search
+41
-23
@@ -16,7 +16,9 @@ The allow-list and the classifier are ported from `openings.go` and
|
||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
||||
its slug is. Every board is read independently; one that fails is logged,
|
||||
recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
||||
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
||||
whose read lists under a tenth of what it listed last time (`_runner.read`'s
|
||||
guard), until it has failed that way for the pull's `stale_days`. A board's
|
||||
filter is the `reach` option the pull hands it (the agents', see
|
||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||
@@ -26,7 +28,8 @@ the whole board: it prunes no per-role state.
|
||||
A board's size (`board_size`) is set once, when the board is added
|
||||
(`app.jobs.sources.set_board`) or rechecked, and kept under
|
||||
`ats:size:<target>` with the day it was checked under
|
||||
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small
|
||||
`ats:sizechecked:<target>` (`keep_size`, which also resets the guard's count,
|
||||
so a Recheck accepts a board that shrank); the reach reads it to leave small
|
||||
boards whole. A pull does not rewrite it. A board with no size yet is sized
|
||||
in its batch before it is read: its reach then says `unsized` and the size
|
||||
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
|
||||
@@ -106,8 +109,9 @@ _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||
|
||||
|
||||
def _limits() -> _runner.Limits:
|
||||
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
|
||||
def _limits(stale_days: int | None = None) -> _runner.Limits:
|
||||
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP,
|
||||
stale_days=stale_days)
|
||||
|
||||
|
||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
@@ -339,8 +343,7 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
if (_runner.declared(module) and module.READS.listing == "paged"
|
||||
and module.READS.complete == "stated"):
|
||||
if module.READS.listing == "paged" and module.READS.complete == "stated":
|
||||
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
|
||||
if isinstance(total, int):
|
||||
return total
|
||||
@@ -348,11 +351,18 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
||||
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
||||
|
||||
|
||||
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
|
||||
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
||||
reset: bool = True) -> None:
|
||||
"""Keep board `target`'s size and the day it was checked (ISO,
|
||||
`today`, else today in UTC)."""
|
||||
`today`, else today in UTC). With `reset`, the default and what a
|
||||
Recheck does, the 10% guard's count goes too (`ats:count:<target>`), so
|
||||
the next read of a board that really shrank is taken as it is instead of
|
||||
failing as suspect; the plugin sizing a board as it reads it passes
|
||||
False, since its count is the read it just stamped."""
|
||||
state[f"ats:size:{target}"] = str(size)
|
||||
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
|
||||
if reset:
|
||||
state.pop(f"ats:count:{target}", None)
|
||||
|
||||
|
||||
failure_reason = _runner.failure_reason
|
||||
@@ -400,14 +410,15 @@ class ATSSource(Source):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
ask, unsized = self._ask(client, target, module)
|
||||
ask, unsized, remote = self._ask(client, target, module)
|
||||
stale_days = self.options.get("stale_days")
|
||||
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
||||
limits=_limits())
|
||||
limits=_limits(int(stale_days) if stale_days else None),
|
||||
remote=remote)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
_runner.record_failure(self.state, failing, exc)
|
||||
return
|
||||
self.state.pop(failing, None)
|
||||
|
||||
def compound(source_id: str) -> str:
|
||||
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
||||
@@ -435,11 +446,13 @@ class ATSSource(Source):
|
||||
yield item_of(row, target, label)
|
||||
# Stamped only once every row went out, so a board that blew up half
|
||||
# way through is neither marked ok nor has a partial id list treated
|
||||
# as the whole of it.
|
||||
# as the whole of it, and it stays failing until then: a row that
|
||||
# fails to store (`app.jobs.pull.run_batch`) leaves the record in place.
|
||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||
[compound(i) for i in listing.ids])
|
||||
self.state.pop(failing, None)
|
||||
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
||||
keep_size(self.state, target, len(listing.ids))
|
||||
keep_size(self.state, target, len(listing.ids), reset=False)
|
||||
if detail_sets is not None:
|
||||
# A complete listing drops the ids it no longer lists.
|
||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||
@@ -447,9 +460,11 @@ class ATSSource(Source):
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
|
||||
def _ask(self, client: httpx.Client, target: str,
|
||||
module: Any) -> tuple[readers.Ask, bool]:
|
||||
module: Any) -> tuple[readers.Ask, bool, bool]:
|
||||
"""What `target` is asked for, from the reach its batch handed it and
|
||||
its params, and whether it still needs a size. A board with no size
|
||||
its params, whether it still needs a size, and whether that reach
|
||||
and params want remote roles (`_runner.remote_wanted`, which the
|
||||
runner compares with what it could ask). A board with no size
|
||||
yet (`unsized` in its reach) whose reach would narrow the read is
|
||||
sized first (`board_size`, kept by `keep_size`), and read whole under
|
||||
`small` roles. One whose read is whole anyway is sized by that read,
|
||||
@@ -457,16 +472,18 @@ class ATSSource(Source):
|
||||
given = self.options.get("reach")
|
||||
params, keys = self.params_for(target), self.options.get("keys")
|
||||
if not isinstance(given, dict) or not given.get("unsized"):
|
||||
return _runner.ask_for(module, params, given, keys=keys), False
|
||||
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
|
||||
ask = _runner.ask_for(module, params, reach or None, keys=keys)
|
||||
return (_runner.ask_for(module, params, given, keys=keys), False,
|
||||
_runner.remote_wanted(module, params, given))
|
||||
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None
|
||||
ask = _runner.ask_for(module, params, reach, keys=keys)
|
||||
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
|
||||
return ask, True
|
||||
return ask, True, _runner.remote_wanted(module, params, reach)
|
||||
size = board_size(target, client=client)
|
||||
keep_size(self.state, target, size)
|
||||
keep_size(self.state, target, size, reset=False)
|
||||
if size < int(given.get("small") or 0):
|
||||
reach = None
|
||||
ask = _runner.ask_for(module, params, None, keys=keys)
|
||||
return ask, False
|
||||
return ask, False, _runner.remote_wanted(module, params, reach)
|
||||
|
||||
|
||||
# ------------------------------------------------------ the per-role reads
|
||||
@@ -567,7 +584,8 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
|
||||
|
||||
def item_of(row: Row, target: str, label: str) -> Item:
|
||||
"""One row of board `target` as the `Item` the store takes: its id
|
||||
compound, `host:slug:id`, as `roles.source_id` has it."""
|
||||
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
|
||||
date is dated now, as `Item` itself would date it."""
|
||||
host, _, slug = target.partition(":")
|
||||
description = row.get("description") or ""
|
||||
return Item(
|
||||
@@ -576,7 +594,7 @@ def item_of(row: Row, target: str, label: str) -> Item:
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted"),
|
||||
published_at=row.get("posted") or datetime.now(UTC),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {
|
||||
|
||||
Reference in new issue
Block a user