Merge board size set once, Recheck, and the Companies line naming what a site does not know (wp-a-size)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:45:27 +02:00
commit 8c66aeb445
11 files changed
+786 -70

No files matched your search

+72 -13
View File
@@ -20,10 +20,18 @@ recorded as failing under `ats:failing:<target>`, and skipped. A board's
filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`) win, narrowed to what its reader declares
(`_runner.ask_for`). An unfiltered read records the board's size under
`ats:size:<target>`, which the reach reads to leave small boards whole. A
read narrowed by the agents' fetch terms is never the whole board: it
records no size and prunes no per-role state.
(`_runner.ask_for`). A read narrowed by the agents' fetch terms is never
the whole board: it prunes no per-role state.
A board's size (`board_size`) is set once, when the board is added
(`app.jobs.sources.set_board`) or rechecked, and kept under
`ats:size:<target>` with the day it was checked under
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small
boards whole. A pull does not rewrite it. A board with no size yet is sized
in its batch before it is read: its reach then says `unsized` and the size
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
whose reader cannot narrow that reach is read whole anyway, and sized by
that read once it is complete, rather than by a second request.
A pull reads listings only. A host that states a role's text on the role's
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
@@ -307,12 +315,46 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no state, just "does
this board answer, what does it list, and is that all of it". For `app
companies add|check`. The listing only, unfiltered, no detail reads.
companies check`. The listing only, unfiltered, no detail reads.
Raises the same way `_fetch_rows` does."""
with httpx.Client() as client:
return _fetch_rows(client, target)
#: Pages read at most to size a board whose host states no total on page 1:
#: 5,000 roles on Teamtailor, 10,000 on HR Manager, far past any board seen.
SIZE_PAGES = 100
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
"""How many roles board `target` lists, unfiltered and without terms: a
whole host's listing, the total a `complete="stated"` host gives on page
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
as `_fetch_rows` does, so a board that does not answer is refused."""
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
if client is None:
with httpx.Client() as own:
return board_size(target, client=own)
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
if (_runner.declared(module) and module.READS.listing == "paged"
and module.READS.complete == "stated"):
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
if isinstance(total, int):
return total
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
"""Keep board `target`'s size and the day it was checked (ISO,
`today`, else today in UTC)."""
state[f"ats:size:{target}"] = str(size)
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
failure_reason = _runner.failure_reason
@@ -358,8 +400,7 @@ class ATSSource(Source):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"),
keys=self.options.get("keys"))
ask, unsized = self._ask(client, target, module)
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
limits=_limits())
except Exception as exc:
@@ -397,18 +438,36 @@ class ATSSource(Source):
# as the whole of it.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
if not (listing.filtered or listing.narrowed):
size = len(listing.ids)
if not listing.complete:
kept = self.state.get(f"ats:size:{target}", "")
size = max(size, int(kept) if kept.isdigit() else 0)
self.state[f"ats:size:{target}"] = str(size)
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
keep_size(self.state, target, len(listing.ids))
if detail_sets is not None:
# A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
if key in self.state:
self.state[key] = json.dumps(sorted(ids))
def _ask(self, client: httpx.Client, target: str,
module: Any) -> tuple[readers.Ask, bool]:
"""What `target` is asked for, from the reach its batch handed it and
its params, and whether it still needs a size. A board with no size
yet (`unsized` in its reach) whose reach would narrow the read is
sized first (`board_size`, kept by `keep_size`), and read whole under
`small` roles. One whose read is whole anyway is sized by that read,
once it is complete."""
given = self.options.get("reach")
params, keys = self.params_for(target), self.options.get("keys")
if not isinstance(given, dict) or not given.get("unsized"):
return _runner.ask_for(module, params, given, keys=keys), False
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
ask = _runner.ask_for(module, params, reach or None, keys=keys)
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
return ask, True
size = board_size(target, client=client)
keep_size(self.state, target, size)
if size < int(given.get("small") or 0):
ask = _runner.ask_for(module, params, None, keys=keys)
return ask, False
# ------------------------------------------------------ the per-role reads