Read role pages in an enrich stage after scoring, best roles first
The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open roles without text that an enabled agent ranks ungated in its best `enrich.top` at `enrich.min_score` or above, best score first, on boards whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`, one batch per board. `plugins.ats.read_board_details` reads them and keeps the per-board state; a read that gave no text is read again after `enrich.retry_days` (finding 18). Filled roles are upserted, which drops their vectors, and embed and score follow; `cascade.step` passes over both when nothing was filled. The pull cascade is pull, embed, score, enrich, embed, score, extract, score, summarise. Saving an agent and changing a company's affinity queue the `enrich` job (enrich, embed, score) behind their rescore; a vote does not. The stage has a switch, `stage:enrich`, over `enrich.enabled`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3adafd8950
commit
77b45a4b0f
10 files changed
+956
-219
No files matched your search
+144
-29
@@ -20,6 +20,10 @@ recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
||||
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
|
||||
filter, narrowed to what its reader declares (`_runner.ask_for`).
|
||||
|
||||
A pull reads listings only. A host that states a role's text on the role's
|
||||
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
||||
scoring, best roles first, through `read_board_details`.
|
||||
|
||||
The limits below are read when a board is read, so a test that replaces one
|
||||
on this module reaches the runner.
|
||||
"""
|
||||
@@ -31,7 +35,7 @@ import logging
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -363,41 +367,152 @@ class ATSSource(Source):
|
||||
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
||||
return f"{host}:{slug}:{source_id}"
|
||||
|
||||
# A host with `detail` states a role's text on the role's own page,
|
||||
# which the `enrich` stage reads after scoring (`read_board_details`),
|
||||
# never the pull. Its rows go out without a description, so the text
|
||||
# a role's page gave stays, and a role read on its own page goes out
|
||||
# without the listing's place and remote as well, so what its page
|
||||
# said stays too (see `_runner.read_details`).
|
||||
rows = listing.rows
|
||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||
detail_sets: tuple[set[str], set[str]] | None = None
|
||||
if hasattr(module, "detail"):
|
||||
detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys)
|
||||
listed = {compound(i) for i in listing.ids} if listing.complete else None
|
||||
rows, detailed, failed = _runner.read_details(
|
||||
module, host, slug, get, rows, detailed, failed, listed=listed,
|
||||
limits=_limits())
|
||||
detail_sets = (detailed, failed)
|
||||
detailed, failed = (_runner.id_set(self.state, key)
|
||||
for key in detail_keys(target)[:2])
|
||||
rows = [row | {"description": ""}
|
||||
| (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {})
|
||||
for row in rows]
|
||||
if listing.complete:
|
||||
listed = {compound(i) for i in listing.ids}
|
||||
detail_sets = (detailed & listed, failed & listed)
|
||||
|
||||
for row in rows:
|
||||
description = row.get("description") or ""
|
||||
yield Item(
|
||||
source_id=compound(row["source_id"]),
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted"),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {
|
||||
"company": label,
|
||||
"function": classify(row.get("department") or "", row["title"]),
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
yield item_of(row, target, label)
|
||||
# Stamped only once every row went out, so a board that blew up half
|
||||
# way through is neither marked ok nor has a partial id list treated
|
||||
# as the whole of it, and a pull stopped half way reads the rest of
|
||||
# its details again.
|
||||
# as the whole of it.
|
||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||
[compound(i) for i in listing.ids])
|
||||
if detail_sets is not None:
|
||||
for key, ids in zip(detail_keys, detail_sets, strict=True):
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
# A complete listing drops the ids it no longer lists.
|
||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||
if key in self.state:
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
|
||||
|
||||
# ------------------------------------------------------ the per-role reads
|
||||
|
||||
#: What a pull withholds for a role already read on its own page: the
|
||||
#: listing's place and remote, which would replace what the page stated.
|
||||
_KEPT_FROM_DETAIL: dict[str, Any] = {"location": "", "places": [], "remote": None}
|
||||
|
||||
#: Days before a role whose own page gave no text is read again: an Oracle
|
||||
#: requisition not yet published, an empty SuccessFactors block. A 404 or a
|
||||
#: 410 counts the same, since the listing closes a role that is gone.
|
||||
EMPTY_RETRY_DAYS = 7
|
||||
|
||||
|
||||
def detail_keys(target: str) -> tuple[str, str, str]:
|
||||
"""The state keys of one board's per-role reads: the ids read, the ids
|
||||
whose read failed, and `{id: date}` of the reads that gave no text."""
|
||||
return (f"ats:detailed:{target}", f"ats:detailfailed:{target}",
|
||||
f"ats:detailempty:{target}")
|
||||
|
||||
|
||||
def _dates(state: Any, key: str) -> dict[str, str]:
|
||||
try:
|
||||
value = json.loads(state.get(key) or "{}")
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def detail_due(state: Any, target: str, ids: list[str], *, now: datetime | None = None,
|
||||
retry_days: int = EMPTY_RETRY_DAYS) -> list[str]:
|
||||
"""Of `ids` (compound, `host:slug:id`, roles without text), those to read
|
||||
on their own page, in the order given: never read, failed, or read with
|
||||
no text at least `retry_days` ago. A role read before its date was kept
|
||||
counts as read that long ago."""
|
||||
detailed = _runner.id_set(state, detail_keys(target)[0])
|
||||
empty = _dates(state, detail_keys(target)[2])
|
||||
cutoff = ((now or datetime.now(UTC)) - timedelta(days=retry_days)).isoformat(
|
||||
timespec="seconds")
|
||||
return [i for i in ids if i not in detailed or empty.get(i, "") <= cutoff]
|
||||
|
||||
|
||||
def read_board_details(state: Any, target: str, rows: list[Row], *, client: httpx.Client,
|
||||
cap: int | None = None, now: datetime | None = None,
|
||||
retry_days: int = EMPTY_RETRY_DAYS) -> list[Row]:
|
||||
"""Read `rows` of one board on their own pages, in the order given with
|
||||
the roles whose read failed last, at most `cap` (`DETAIL_CAP`), and keep the board's per-role state
|
||||
(`detail_keys`). The rows are the stored roles without text, each with
|
||||
its host's own `source_id`. Returns the rows whose page filled a field,
|
||||
each merged with what it filled.
|
||||
|
||||
A read that gave no text is kept with its date and read again once it is
|
||||
`retry_days` old (`detail_due`). The sets are read again just before they
|
||||
are written and changed by what this read did, so a pull that dropped ids
|
||||
meanwhile keeps its drop."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
keys = detail_keys(target)
|
||||
now = now or datetime.now(UTC)
|
||||
|
||||
def compound(row: Row) -> str:
|
||||
return f"{host}:{slug}:{row['source_id']}"
|
||||
|
||||
due = set(detail_due(state, target, [compound(r) for r in rows], now=now,
|
||||
retry_days=retry_days))
|
||||
detailed, failed = (_runner.id_set(state, key) for key in keys[:2])
|
||||
# Roles whose read failed go after the rest, so they cannot starve them.
|
||||
wanted = sorted((row for row in rows if compound(row) in due),
|
||||
key=lambda row: compound(row) in failed)
|
||||
wanted = wanted[:DETAIL_CAP if cap is None else cap]
|
||||
if not wanted:
|
||||
return []
|
||||
again = {compound(row) for row in wanted} & detailed
|
||||
limits = _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=len(wanted))
|
||||
out, done, failing = _runner.read_details(
|
||||
module, host, slug, _getter(client, host, slug), wanted, detailed - again, failed,
|
||||
limits=limits)
|
||||
read = done - (detailed - again)
|
||||
filled: list[Row] = []
|
||||
empty = _dates(state, keys[2])
|
||||
for before, after in zip(wanted, out, strict=True):
|
||||
key = compound(before)
|
||||
if key in read:
|
||||
if after.get("description"):
|
||||
empty.pop(key, None)
|
||||
else:
|
||||
empty[key] = now.isoformat(timespec="seconds")
|
||||
if after != before | {"description": ""}:
|
||||
filled.append(after)
|
||||
|
||||
detailed_now = (_runner.id_set(state, keys[0]) - again) | read
|
||||
failed_now = (_runner.id_set(state, keys[1]) | (failing - failed)) - read
|
||||
state[keys[0]] = json.dumps(sorted(detailed_now))
|
||||
state[keys[1]] = json.dumps(sorted(failed_now))
|
||||
state[keys[2]] = json.dumps({k: v for k, v in sorted(empty.items()) if k in detailed_now})
|
||||
return filled
|
||||
|
||||
|
||||
def item_of(row: Row, target: str, label: str) -> Item:
|
||||
"""One row of board `target` as the `Item` the store takes: its id
|
||||
compound, `host:slug:id`, as `roles.source_id` has it."""
|
||||
host, _, slug = target.partition(":")
|
||||
description = row.get("description") or ""
|
||||
return Item(
|
||||
source_id=f"{host}:{slug}:{row['source_id']}",
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted"),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {
|
||||
"company": label,
|
||||
"function": classify(row.get("department") or "", row["title"]),
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
Reference in new issue
Block a user