Merge board size set once, Recheck, and the Companies line naming what a site does not know (wp-a-size)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
8c66aeb445
11 files changed
+786
-70
No files matched your search
+72
-13
@@ -20,10 +20,18 @@ recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
||||
filter is the `reach` option the pull hands it (the agents', see
|
||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||
(`_runner.ask_for`). An unfiltered read records the board's size under
|
||||
`ats:size:<target>`, which the reach reads to leave small boards whole. A
|
||||
read narrowed by the agents' fetch terms is never the whole board: it
|
||||
records no size and prunes no per-role state.
|
||||
(`_runner.ask_for`). A read narrowed by the agents' fetch terms is never
|
||||
the whole board: it prunes no per-role state.
|
||||
|
||||
A board's size (`board_size`) is set once, when the board is added
|
||||
(`app.jobs.sources.set_board`) or rechecked, and kept under
|
||||
`ats:size:<target>` with the day it was checked under
|
||||
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small
|
||||
boards whole. A pull does not rewrite it. A board with no size yet is sized
|
||||
in its batch before it is read: its reach then says `unsized` and the size
|
||||
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
|
||||
whose reader cannot narrow that reach is read whole anyway, and sized by
|
||||
that read once it is complete, rather than by a second request.
|
||||
|
||||
A pull reads listings only. A host that states a role's text on the role's
|
||||
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
||||
@@ -307,12 +315,46 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no state, just "does
|
||||
this board answer, what does it list, and is that all of it". For `app
|
||||
companies add|check`. The listing only, unfiltered, no detail reads.
|
||||
companies check`. The listing only, unfiltered, no detail reads.
|
||||
Raises the same way `_fetch_rows` does."""
|
||||
with httpx.Client() as client:
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
#: Pages read at most to size a board whose host states no total on page 1:
|
||||
#: 5,000 roles on Teamtailor, 10,000 on HR Manager, far past any board seen.
|
||||
SIZE_PAGES = 100
|
||||
|
||||
|
||||
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
||||
"""How many roles board `target` lists, unfiltered and without terms: a
|
||||
whole host's listing, the total a `complete="stated"` host gives on page
|
||||
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
|
||||
as `_fetch_rows` does, so a board that does not answer is refused."""
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
if client is None:
|
||||
with httpx.Client() as own:
|
||||
return board_size(target, client=own)
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
if (_runner.declared(module) and module.READS.listing == "paged"
|
||||
and module.READS.complete == "stated"):
|
||||
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
|
||||
if isinstance(total, int):
|
||||
return total
|
||||
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
|
||||
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
||||
|
||||
|
||||
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
|
||||
"""Keep board `target`'s size and the day it was checked (ISO,
|
||||
`today`, else today in UTC)."""
|
||||
state[f"ats:size:{target}"] = str(size)
|
||||
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
|
||||
|
||||
|
||||
failure_reason = _runner.failure_reason
|
||||
|
||||
|
||||
@@ -358,8 +400,7 @@ class ATSSource(Source):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"),
|
||||
keys=self.options.get("keys"))
|
||||
ask, unsized = self._ask(client, target, module)
|
||||
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
||||
limits=_limits())
|
||||
except Exception as exc:
|
||||
@@ -397,18 +438,36 @@ class ATSSource(Source):
|
||||
# as the whole of it.
|
||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||
[compound(i) for i in listing.ids])
|
||||
if not (listing.filtered or listing.narrowed):
|
||||
size = len(listing.ids)
|
||||
if not listing.complete:
|
||||
kept = self.state.get(f"ats:size:{target}", "")
|
||||
size = max(size, int(kept) if kept.isdigit() else 0)
|
||||
self.state[f"ats:size:{target}"] = str(size)
|
||||
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
||||
keep_size(self.state, target, len(listing.ids))
|
||||
if detail_sets is not None:
|
||||
# A complete listing drops the ids it no longer lists.
|
||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||
if key in self.state:
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
|
||||
def _ask(self, client: httpx.Client, target: str,
|
||||
module: Any) -> tuple[readers.Ask, bool]:
|
||||
"""What `target` is asked for, from the reach its batch handed it and
|
||||
its params, and whether it still needs a size. A board with no size
|
||||
yet (`unsized` in its reach) whose reach would narrow the read is
|
||||
sized first (`board_size`, kept by `keep_size`), and read whole under
|
||||
`small` roles. One whose read is whole anyway is sized by that read,
|
||||
once it is complete."""
|
||||
given = self.options.get("reach")
|
||||
params, keys = self.params_for(target), self.options.get("keys")
|
||||
if not isinstance(given, dict) or not given.get("unsized"):
|
||||
return _runner.ask_for(module, params, given, keys=keys), False
|
||||
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
|
||||
ask = _runner.ask_for(module, params, reach or None, keys=keys)
|
||||
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
|
||||
return ask, True
|
||||
size = board_size(target, client=client)
|
||||
keep_size(self.state, target, size)
|
||||
if size < int(given.get("small") or 0):
|
||||
ask = _runner.ask_for(module, params, None, keys=keys)
|
||||
return ask, False
|
||||
|
||||
|
||||
# ------------------------------------------------------ the per-role reads
|
||||
|
||||
|
||||
Reference in new issue
Block a user