Read role pages in an enrich stage after scoring, best roles first

The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open
roles without text that an enabled agent ranks ungated in its best
`enrich.top` at `enrich.min_score` or above, best score first, on boards
whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`,
one batch per board. `plugins.ats.read_board_details` reads them and keeps
the per-board state; a read that gave no text is read again after
`enrich.retry_days` (finding 18). Filled roles are upserted, which drops
their vectors, and embed and score follow; `cascade.step` passes over both
when nothing was filled.

The pull cascade is pull, embed, score, enrich, embed, score, extract, score,
summarise. Saving an agent and changing a company's affinity queue the
`enrich` job (enrich, embed, score) behind their rescore; a vote does not.
The stage has a switch, `stage:enrich`, over `enrich.enabled`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:53:29 +02:00
1 parent 3adafd8950
commit 77b45a4b0f
10 files changed
+956 -219

No files matched your search

+20
View File
@@ -66,6 +66,25 @@ class SummariseConfig(ModelTaskConfig):
batch_size: int = Field(10, ge=1)
class EnrichConfig(Section):
"""Reading roles on their own pages after scoring, for the hosts whose
list carries no text. See app/jobs/enrich.py. A role is read when it has
no text and an enabled agent with a prompt ranks it ungated among its
best `top` and at `min_score` or above: the extract bar (`Limits`,
500 and 0.55) with the score a notch lower, since a role ranked on its
title alone is the one whose score its text moves most."""
enabled: bool = True
top: int = Field(500, ge=0)
min_score: float = Field(0.5, ge=0, le=1)
#: Reads per board and per run. Per board is the cap the pull had; the
#: boards are read in parallel on the `http` pool.
per_board: int = Field(200, ge=0)
per_run: int = Field(1000, ge=0)
#: Days before a page that gave no text is read again.
retry_days: int = Field(7, ge=1)
class Calibration(Section):
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
@@ -181,6 +200,7 @@ class Config(Section):
extract: ExtractConfig = Field(default_factory=ExtractConfig)
summarise: SummariseConfig = Field(default_factory=SummariseConfig)
embed: EmbedConfig = Field(default_factory=EmbedConfig)
enrich: EnrichConfig = Field(default_factory=EnrichConfig)
#: Starter agents, imported once into an empty table.
agents: list[AgentSeed] = Field(default_factory=list)
#: Starter searches, imported once into an empty set of `search`
+44 -5
View File
@@ -1,6 +1,6 @@
"""The tasks roledesk offers, and the order a pull runs them in.
pull → embed → score → extract → score → summarise
pull → embed → score → enrich → embed → score → extract → score → summarise
newsdesk's cascade convention: a cascade is a tuple of task names, and the
final job of one task starts the next (`step` is the callback it carries).
@@ -11,7 +11,10 @@ Every stage is always in the chain, and a stage that is switched off finds
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
satisfy, is an extract stage with no batches whose final job moves straight
on. The free stages go first: embed and score cost no tokens, and the score
they produce is what bounds the two that do. Extract reads only the best
they produce is what bounds the rest. Enrich reads the best roles without
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
the roles it filled; when it filled none, `step` passes over both. Extract
reads only the best
roles of each enabled agent, as deep and as low as that agent's own `limits`
say; the second score then applies the gates to the columns it filled.
Summarise runs last, on the best roles by the same agent's summarise limits.
@@ -23,6 +26,11 @@ having either way.
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
and a cascade is started by one: the `pull` job's only work is to ask for the
chain. Its default schedule, `every 6h`, is declared here beside it.
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
behind the `score` that follows a change to the ranking other than a pull's
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
one agent's taste a little and queues neither.
"""
from __future__ import annotations
@@ -32,16 +40,21 @@ from dataclasses import replace
from functools import partial
from typing import Any
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
CASCADES: dict[str, tuple[str, ...]] = {
"pull": ("pull", "embed", "score", "extract", "score", "summarise"),
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
"summarise"),
"enrich": ("enrich", "embed", "score"),
}
#: What follows `enrich` for the roles it filled, passed over when it filled none.
REFILL = ("embed", "score")
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
"""One batch with nothing in its payload: a task that is a single job."""
@@ -91,6 +104,7 @@ STAGES: dict[str, Callable[[str], Task]] = {
"score": lambda cascade: Task(kind="score", finder=_once,
label="Scoring roles for every agent"),
"summarise": _summarise,
"enrich": lambda cascade: enrich.enrich_task(),
}
@@ -115,6 +129,10 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
chain goes on, as in newsdesk's cascade."""
name, done = payload.get("cascade", ""), payload.get("task", "")
at = int(payload.get("at", -1))
tasks = CASCADES.get(name, ())
if done == "enrich" and not enrich.filled(tally) \
and tasks[at + 1:at + 1 + len(REFILL)] == REFILL:
at += len(REFILL)
following = follows(name, at)
if following:
context.task(task(following, cascade=name, at=at + 1))
@@ -169,6 +187,26 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
return {"started": " > ".join(CASCADES["pull"]), "run": run}
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
if not context.task(chain("enrich")):
raise RuntimeError("no queue to start the enrich cascade on")
return {"started": " > ".join(CASCADES["enrich"])}
def enrich_after(queue: Any, score_job: Any) -> Any:
"""Queue the `enrich` job behind `score_job`, the rescore that follows a
change to the ranking, however that score ends. One per score job
(`enrich:after:<id>`), so ten changes answered by one score are one
enrich. None without a score job, or when the queue refuses."""
if score_job is None:
return None
return queue.enqueue("enrich", depends_on=[score_job.id], after="done",
dedup_key=f"enrich:after:{score_job.id}")
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be listed twice by `/api/jobs/kinds`.
@@ -182,4 +220,5 @@ for _name in ("extract", "embed", "score", "summarise"):
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then embed, score, extract, rescore and summarise")
label="Pull every board, then embed, score, read the best roles' pages, "
"extract, rescore and summarise")
+168
View File
@@ -0,0 +1,168 @@
"""`enrich`: the best roles without text, read on their own pages.
Some hosts list a role without its text and state it on the role's own page
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
with a `detail`). The pull reads listings only; this stage reads those pages
after scoring, so the calls go where the ranking is:
1. `enrich_waiting` hands over the open roles without text that an enabled
agent with a prompt ranks ungated among its best `enrich.top` and at
`enrich.min_score` or above, best first by the highest such score. A role
below that bar for every agent gets no call.
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
out the roles already read, unless their read failed or gave no text
`enrich.retry_days` ago (`plugins.ats.detail_due`).
3. One batch per board, on the `http` pool, reads them
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
what each page filled. `Store.upsert_role` drops the vector of a role
whose text improved, so the next `embed` encodes those roles only.
In the pull cascade `embed` and `score` follow; `cascade.step` passes over
both when no role was filled. The same stage follows a rescore after an agent
is saved or a company's affinity changes (`cascade.enrich_after`), never after
a vote. Whether it runs is `stages.stage_on(config, store, "enrich")`.
"""
from __future__ import annotations
import json
import logging
from collections import defaultdict
from datetime import datetime
from typing import Any
import httpx
from app.config import Config
from app.jobs import stages
from app.storage import Store
from pipeline import NULL, Batch, RunContext, Tally, Task, job
log = logging.getLogger(__name__)
KIND = "enrich_batch"
SETTLE = "app.jobs.enrich:settle"
def _ats() -> Any:
"""`plugins.ats`, imported when needed, the way `pull.run_batch` does."""
from plugins import ats
return ats
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
"""A stored role back as the row its reader listed: the stored `raw`
with the host's own id, the title, the link and the date."""
try:
raw = json.loads(found.get("raw") or "{}")
except ValueError:
raw = {}
raw = raw if isinstance(raw, dict) else {}
try:
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
except ValueError:
posted = None
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
"title": found["title"], "url": found["url"], "description": "",
"posted": posted}
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
"""`(target, uids)` per board, best board first, within the budgets: the
roles `enrich_waiting` ranks, on fetchable boards whose reader has
`detail` and that are not failing, and due for a read."""
settings = config.enrich
ats = _ats()
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
for found in store.enrich_waiting(settings.top, settings.min_score):
target = boards.get(found["subscription_id"])
if target is None:
continue
module = ats.readers.HOSTS.get(target.partition(":")[0])
if module is None or not hasattr(module, "detail") \
or f"ats:failing:{target}" in store.state:
continue
by_board[target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
retry_days=settings.retry_days))
for target, found in by_board.items()}
# Best first; roles whose read failed after every other, so a page that
# keeps failing cannot take the budget of the roles never tried.
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
for target in by_board}
ranked = sorted((f for found in by_board.values() for f in found),
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
-float(f["score"] or 0), f["uid"]))
picked: dict[str, list[str]] = {}
total = 0
for found in ranked:
target = boards[found["subscription_id"]]
if found["source_id"] not in due[target]:
continue
if total >= settings.per_run:
break
uids = picked.setdefault(target, [])
if len(uids) >= settings.per_board:
continue
uids.append(found["uid"])
total += 1
return list(picked.items())
def batches(config: Config, store: Store) -> list[Batch]:
"""The finder: one batch per board, nothing while the stage is off."""
if not stages.stage_on(config, store, "enrich"):
return []
return [Batch({"target": target, "uids": uids},
label=f"Reading {len(uids)} roles on {target}")
for target, uids in plan(config, store)]
@job(KIND, "Reading roles on their own pages", resource="http")
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Read one board's roles in `uids` on their own pages, in that order,
and store what each page filled. `filled` is how many roles changed,
`texts` how many of them now have text."""
ats = _ats()
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
if not subs or not uids:
return {"target": target, "filled": 0, "texts": 0}
sub = subs[0]
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
rows = [_row(found[uid], target) for uid in uids if uid in found]
context.check()
with httpx.Client() as client:
filled = ats.read_board_details(store.state, target, rows, client=client,
cap=len(rows), retry_days=config.enrich.retry_days)
label = sub.company or sub.label
texts = 0
for row in filled:
item = ats.item_of(row, target, label)
item.source = "ats"
store.upsert_role(item, sub)
texts += bool(row.get("description"))
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
texts)
return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts}
def filled(tally: Tally) -> int:
"""How many roles the batches behind `tally` filled."""
return sum(int(result.get("filled") or 0) for result in tally.results)
def settle(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's sum: roles filled, and how many of them now have text."""
return {"filled": filled(tally),
"texts": sum(int(result.get("texts") or 0) for result in tally.results)}
def enrich_task() -> Task:
return Task(kind=KIND, finder=batches, name="enrich", settle=SETTLE,
label="Reading the best roles on their own pages")
+16 -7
View File
@@ -1,4 +1,5 @@
"""Whether the two model stages of a pull, `extract` and `summarise`, run.
"""Whether the two model stages of a pull, `extract` and `summarise`, and
the `enrich` stage run.
`extract.enabled` and `summarise.enabled` in config.yaml are read at startup
and say nothing on the page, so the switch lives in the database as well: the
@@ -8,9 +9,13 @@ default for a fresh database and a deployment that never uses the switch
behaves as it did. `stage_on` is the one place that decides, and both tasks'
finders ask it (`extract.extract_task`, `summarise.summarise_task`).
`describe` is what `GET /api/stages` and `app stages` show: for each stage
whether it is on, where that comes from, the model it uses, and who would
answer.
`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`)
asks no model and is on by default (`enrich.enabled`); its switch is
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
`describe` is what `GET /api/stages` and `app stages` show: for each model
stage whether it is on, where that comes from, the model it uses, and who
would answer.
"""
from __future__ import annotations
@@ -25,6 +30,9 @@ from desk.tools import agent
NAMES = ("extract", "summarise")
#: Every stage with a switch: the model stages and `enrich`.
SWITCHES = (*NAMES, "enrich")
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
"see docs/llm-backends.md")
ANSWERS = {
@@ -39,7 +47,7 @@ class Stage(BaseModel):
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
it in words."""
name: Literal["extract", "summarise"]
name: Literal["extract", "summarise", "enrich"]
on: bool
source: Literal["stored", "config"]
model: str
@@ -65,7 +73,7 @@ def stage_on(config: Config, store: Store, name: str) -> bool:
def set_stage(store: Store, name: str, on: bool) -> None:
"""Store the switch. A `KeyError` for a name that is not a stage."""
if name not in NAMES:
if name not in SWITCHES:
raise KeyError(name)
store.state[_key(name)] = "on" if on else "off"
@@ -84,7 +92,8 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
backend, said = answer(config)
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
model=getattr(config, name).model, backend=backend, answer=said)
model=getattr(getattr(config, name), "model", ""), backend=backend,
answer=said)
def describe(config: Config, store: Store) -> list[Stage]:
+40
View File
@@ -1398,6 +1398,46 @@ class Store:
return None
return self.role(uid)
# --------------------------------------------------------------- enrich
def enrich_waiting(self, top: int, floor: float,
uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
"""Open `ats` roles without text that an enabled agent with a prompt
ranks ungated among its best `top` and at `floor` or above, best
first by the highest such score, as `{uid, source_id,
subscription_id, title, url, published_at, raw, score}`. The ranking
is `_top_scope`'s, with the bar given rather than the agent's own.
Given `uids`, those of them still without text, whatever their rank,
in no order (`score` is None), since a batch finishes what it was
dealt."""
fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, "
"r.published_at, r.raw")
waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source = 'ats' "
"AND r.subscription_id IS NOT NULL AND r.description = '' "
"AND r.article_text = ''")
if uids is not None:
rows = self._rows(
f"SELECT {fields}, NULL AS score FROM roles r WHERE {waiting} "
"AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),))
else:
rows = self._rows(
f"""SELECT {fields}, MAX(ranked.score) AS score FROM (
SELECT s.uid, s.score, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
AND EXISTS (SELECT 1 FROM agent_prompts p
WHERE p.agent_id = a.id
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
AND top.dup_of IS NULL
WHERE s.gated_reason = '') ranked
JOIN roles r ON r.uid = ranked.uid
WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting}
GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor))
return [dict(row) for row in rows]
# -------------------------------------------------------------- vectors
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
+9 -6
View File
@@ -679,6 +679,7 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep,
except KeyError:
raise HTTPException(404, f"no such company: {key}") from None
job = _score_after_change(queue)
jobs.cascade.enrich_after(queue, job)
return AffinityOut(**company.model_dump(), score_job=job.id if job else None)
@@ -963,17 +964,19 @@ class AgentBody(BaseModel):
@router.post("/agents")
def save_agent(body: AgentBody, store: StoreDep) -> Agent:
"""Scores are not recomputed here: the page asks `POST /api/score` after
a save and waits on the job it answers with."""
def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent:
"""Queues the rescore, which the page's `POST /api/score` after a save
joins, and the enrich run behind it (`cascade.enrich_after`)."""
try:
return store.save_agent(body.name, body.intent, body.gates, body.weights,
body.enabled, agent_id=body.id, limits=body.limits,
prompts=body.prompts)
saved = store.save_agent(body.name, body.intent, body.gates, body.weights,
body.enabled, agent_id=body.id, limits=body.limits,
prompts=body.prompts)
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
except KeyError:
raise HTTPException(404, f"no such agent: {body.id}") from None
jobs.cascade.enrich_after(queue, _score_after_change(queue))
return saved
@router.delete("/agents/{agent_id}", status_code=204)
+144 -29
View File
@@ -20,6 +20,10 @@ recorded as failing under `ats:failing:<target>`, and skipped. A board's
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
filter, narrowed to what its reader declares (`_runner.ask_for`).
A pull reads listings only. A host that states a role's text on the role's
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
scoring, best roles first, through `read_board_details`.
The limits below are read when a board is read, so a test that replaces one
on this module reaches the runner.
"""
@@ -31,7 +35,7 @@ import logging
import re
import sys
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -363,41 +367,152 @@ class ATSSource(Source):
# Matches `roles.source_id`, which `Store.mark_closed` compares.
return f"{host}:{slug}:{source_id}"
# A host with `detail` states a role's text on the role's own page,
# which the `enrich` stage reads after scoring (`read_board_details`),
# never the pull. Its rows go out without a description, so the text
# a role's page gave stays, and a role read on its own page goes out
# without the listing's place and remote as well, so what its page
# said stays too (see `_runner.read_details`).
rows = listing.rows
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(module, "detail"):
detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys)
listed = {compound(i) for i in listing.ids} if listing.complete else None
rows, detailed, failed = _runner.read_details(
module, host, slug, get, rows, detailed, failed, listed=listed,
limits=_limits())
detail_sets = (detailed, failed)
detailed, failed = (_runner.id_set(self.state, key)
for key in detail_keys(target)[:2])
rows = [row | {"description": ""}
| (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {})
for row in rows]
if listing.complete:
listed = {compound(i) for i in listing.ids}
detail_sets = (detailed & listed, failed & listed)
for row in rows:
description = row.get("description") or ""
yield Item(
source_id=compound(row["source_id"]),
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": label,
"function": classify(row.get("department") or "", row["title"]),
"board": host,
"slug": slug,
},
)
yield item_of(row, target, label)
# Stamped only once every row went out, so a board that blew up half
# way through is neither marked ok nor has a partial id list treated
# as the whole of it, and a pull stopped half way reads the rest of
# its details again.
# as the whole of it.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
if detail_sets is not None:
for key, ids in zip(detail_keys, detail_sets, strict=True):
self.state[key] = json.dumps(sorted(ids))
# A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
if key in self.state:
self.state[key] = json.dumps(sorted(ids))
# ------------------------------------------------------ the per-role reads
#: What a pull withholds for a role already read on its own page: the
#: listing's place and remote, which would replace what the page stated.
_KEPT_FROM_DETAIL: dict[str, Any] = {"location": "", "places": [], "remote": None}
#: Days before a role whose own page gave no text is read again: an Oracle
#: requisition not yet published, an empty SuccessFactors block. A 404 or a
#: 410 counts the same, since the listing closes a role that is gone.
EMPTY_RETRY_DAYS = 7
def detail_keys(target: str) -> tuple[str, str, str]:
"""The state keys of one board's per-role reads: the ids read, the ids
whose read failed, and `{id: date}` of the reads that gave no text."""
return (f"ats:detailed:{target}", f"ats:detailfailed:{target}",
f"ats:detailempty:{target}")
def _dates(state: Any, key: str) -> dict[str, str]:
try:
value = json.loads(state.get(key) or "{}")
except (TypeError, ValueError):
return {}
return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {}
def detail_due(state: Any, target: str, ids: list[str], *, now: datetime | None = None,
retry_days: int = EMPTY_RETRY_DAYS) -> list[str]:
"""Of `ids` (compound, `host:slug:id`, roles without text), those to read
on their own page, in the order given: never read, failed, or read with
no text at least `retry_days` ago. A role read before its date was kept
counts as read that long ago."""
detailed = _runner.id_set(state, detail_keys(target)[0])
empty = _dates(state, detail_keys(target)[2])
cutoff = ((now or datetime.now(UTC)) - timedelta(days=retry_days)).isoformat(
timespec="seconds")
return [i for i in ids if i not in detailed or empty.get(i, "") <= cutoff]
def read_board_details(state: Any, target: str, rows: list[Row], *, client: httpx.Client,
cap: int | None = None, now: datetime | None = None,
retry_days: int = EMPTY_RETRY_DAYS) -> list[Row]:
"""Read `rows` of one board on their own pages, in the order given with
the roles whose read failed last, at most `cap` (`DETAIL_CAP`), and keep the board's per-role state
(`detail_keys`). The rows are the stored roles without text, each with
its host's own `source_id`. Returns the rows whose page filled a field,
each merged with what it filled.
A read that gave no text is kept with its date and read again once it is
`retry_days` old (`detail_due`). The sets are read again just before they
are written and changed by what this read did, so a pull that dropped ids
meanwhile keeps its drop."""
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
keys = detail_keys(target)
now = now or datetime.now(UTC)
def compound(row: Row) -> str:
return f"{host}:{slug}:{row['source_id']}"
due = set(detail_due(state, target, [compound(r) for r in rows], now=now,
retry_days=retry_days))
detailed, failed = (_runner.id_set(state, key) for key in keys[:2])
# Roles whose read failed go after the rest, so they cannot starve them.
wanted = sorted((row for row in rows if compound(row) in due),
key=lambda row: compound(row) in failed)
wanted = wanted[:DETAIL_CAP if cap is None else cap]
if not wanted:
return []
again = {compound(row) for row in wanted} & detailed
limits = _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=len(wanted))
out, done, failing = _runner.read_details(
module, host, slug, _getter(client, host, slug), wanted, detailed - again, failed,
limits=limits)
read = done - (detailed - again)
filled: list[Row] = []
empty = _dates(state, keys[2])
for before, after in zip(wanted, out, strict=True):
key = compound(before)
if key in read:
if after.get("description"):
empty.pop(key, None)
else:
empty[key] = now.isoformat(timespec="seconds")
if after != before | {"description": ""}:
filled.append(after)
detailed_now = (_runner.id_set(state, keys[0]) - again) | read
failed_now = (_runner.id_set(state, keys[1]) | (failing - failed)) - read
state[keys[0]] = json.dumps(sorted(detailed_now))
state[keys[1]] = json.dumps(sorted(failed_now))
state[keys[2]] = json.dumps({k: v for k, v in sorted(empty.items()) if k in detailed_now})
return filled
def item_of(row: Row, target: str, label: str) -> Item:
"""One row of board `target` as the `Item` the store takes: its id
compound, `host:slug:id`, as `roles.source_id` has it."""
host, _, slug = target.partition(":")
description = row.get("description") or ""
return Item(
source_id=f"{host}:{slug}:{row['source_id']}",
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": label,
"function": classify(row.get("department") or "", row["title"]),
"board": host,
"slug": slug,
},
)
+368
View File
@@ -0,0 +1,368 @@
"""The `enrich` stage: the best roles without text, read on their own pages
after scoring. Which roles and in what order, the budgets, what is stored,
the cascade around it, and what triggers it. No network: a fake host stands
in, answering from a dict."""
from __future__ import annotations
import json
import sys
import types
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import httpx
import numpy as np
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.config import Config, ServerConfig # noqa: E402
from app.jobs import cascade, embed, enrich, stages # noqa: E402
from app.main import create_app # noqa: E402
from app.storage import Store # noqa: E402
from desk.models import Item # noqa: E402
from pipeline import FINAL, JobQueue, Jobs, Tally # noqa: E402
from plugins import ats, readers # noqa: E402
BASE = "https://jobs.fake.example"
def _fake_host(name: str = "fake", *, detail: bool = True) -> types.ModuleType:
"""A host whose role pages are `BASE/<slug>/<id>`, answered by `web`."""
module: Any = types.ModuleType(name)
module.NAME, module.LABEL, module.PATTERNS = name, name.title(), ()
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"{BASE}/{slug}"
module.feed_url = lambda slug: f"{BASE}/{slug}/list"
module.parse = lambda body, slug: []
if detail:
module.detail = lambda get, slug, row: {
"description": get(f"{BASE}/{slug}/{row['source_id']}").decode(),
"location": "Copenhagen"}
return module
@pytest.fixture
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
"""`ats._get` answering from `answers` (URL -> body), 404 otherwise,
remembering what it was asked; hosts `fake` and `plain` (no detail)."""
answers: dict[str, Any] = {}
asked: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
asked.append(url)
answer = answers.get(url)
if answer is None:
request = httpx.Request("GET", url)
raise httpx.HTTPStatusError("404", request=request,
response=httpx.Response(404, request=request))
return answer
monkeypatch.setattr(ats, "_get", fake_get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host())
monkeypatch.setitem(readers.HOSTS, "plain", _fake_host("plain", detail=False))
return {"answers": answers, "asked": asked}
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def config(tmp_path: Path, **enrich_settings: Any) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"},
enrich=enrich_settings)
def board(store: Store, target: str) -> Any:
host = target.partition(":")[0]
return store.add_subscription("ats", target, label=target, company=target, board=host)
def role(store: Store, target: str, native: str, *, body: str = "",
title: str = "Backend Engineer") -> str:
"""A role of board `target` as a pull stores it: without text on a host
with `detail`."""
sub = next(s for s in store.subscriptions(source="ats") if s.target == target)
host, _, slug = target.partition(":")
item = Item(source_id=f"{target}:{native}", location=f"{BASE}/{slug}/{native}",
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": target, "place": "2 Locations", "board": host, "slug": slug})
item.source = "ats"
store.upsert_role(item, sub)
return item.uid
def scored(store: Store, name: str, scores: dict[str, float], *,
gated: tuple[str, ...] = ()) -> Any:
agent = store.save_agent(name, f"{name} python backend")
store.replace_scores(agent.id, [(uid, value, "gated" if uid in gated else "")
for uid, value in scores.items()])
return agent
# -------------------------------------------------------------- which roles
def test_roles_go_best_first_by_their_best_agent_score(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
board(store, "fake:beta")
a1, a2 = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
b1 = role(store, "fake:beta", "1")
scored(store, "One", {a1: 0.6, a2: 0.9, b1: 0.55})
scored(store, "Two", {a1: 0.95, b1: 0.7})
assert [(r["uid"], r["score"]) for r in store.enrich_waiting(500, 0.5)] == [
(a1, 0.95), (a2, 0.9), (b1, 0.7)]
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]),
("fake:beta", [b1])]
def test_a_role_no_agent_ranks_high_enough_gets_no_call(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
low, gated, deep, kept = (role(store, "fake:acme", n) for n in ("1", "2", "3", "4"))
scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,))
# `top` 1 per agent: only the best ungated role of each agent is in reach.
assert enrich.plan(config(tmp_path, top=1), store) == [("fake:acme", [kept])]
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])]
# A disabled agent ranks nothing.
agent = store.agents()[0]
store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, enabled=False,
agent_id=agent.id, limits=agent.limits, prompts=agent.prompts)
assert enrich.plan(config(tmp_path), store) == []
def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
for target in ("fake:acme", "plain:acme", "fake:down"):
board(store, target)
wanted = role(store, "fake:acme", "1")
has_text = role(store, "fake:acme", "2", body="Python in full")
plain = role(store, "plain:acme", "1")
down = role(store, "fake:down", "1")
store.state["ats:failing:fake:down"] = json.dumps({"since": "2026-09-29", "reason": "500"})
scored(store, "One", {wanted: 0.9, has_text: 0.95, plain: 0.9, down: 0.9})
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [wanted])]
def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
read, failed, fresh = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
now = datetime.now(UTC).isoformat(timespec="seconds")
store.state["ats:detailed:fake:acme"] = json.dumps(["fake:acme:1"])
store.state["ats:detailempty:fake:acme"] = json.dumps({"fake:acme:1": now})
store.state["ats:detailfailed:fake:acme"] = json.dumps(["fake:acme:2"])
scored(store, "One", {read: 0.9, failed: 0.8, fresh: 0.6})
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])]
# ------------------------------------------------------------------ budgets
def test_the_budgets_per_board_and_per_run(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
board(store, "fake:beta")
acme = [role(store, "fake:acme", str(n)) for n in range(4)]
beta = [role(store, "fake:beta", str(n)) for n in range(4)]
# Interleaved: acme 0.99, beta 0.98, acme 0.97, ...
scored(store, "One", {uid: 0.99 - 0.01 * n
for n, uid in enumerate(u for pair in zip(acme, beta, strict=True)
for u in pair)})
assert enrich.plan(config(tmp_path, per_board=2), store) == [
("fake:acme", acme[:2]), ("fake:beta", beta[:2])]
assert enrich.plan(config(tmp_path, per_run=3), store) == [
("fake:acme", acme[:2]), ("fake:beta", beta[:1])]
assert enrich.plan(config(tmp_path, per_board=1, per_run=5), store) == [
("fake:acme", acme[:1]), ("fake:beta", beta[:1])]
def test_the_stage_off_finds_nothing(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
scored(store, "One", {role(store, "fake:acme", "1"): 0.9})
assert len(enrich.batches(config(tmp_path), store)) == 1
assert enrich.batches(config(tmp_path, enabled=False), store) == []
stages.set_stage(store, "enrich", True)
assert len(enrich.batches(config(tmp_path, enabled=False), store)) == 1
stages.set_stage(store, "enrich", False)
assert enrich.batches(config(tmp_path), store) == []
# ------------------------------------------------------------ what is stored
def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
one, two, gone = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32)) for uid in (one, two, gone)])
web["answers"].update({f"{BASE}/acme/1": b"Python in full", f"{BASE}/acme/2": b"Go too"})
out = enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one])
assert out == {"target": "fake:acme", "read": 3, "filled": 2, "texts": 2}
assert web["asked"] == [f"{BASE}/acme/2", f"{BASE}/acme/3", f"{BASE}/acme/1"]
first = store.role(one)
assert first is not None and first.description == "Python in full"
assert first.place == "Copenhagen" and first.title == "Backend Engineer"
assert set(store.vectors("fake")) == {gone}
assert json.loads(store.state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
# A later pull of the same listing keeps the text and the page's place.
item = ats.item_of({"source_id": "1", "title": "Backend Engineer", "url": f"{BASE}/acme/1",
"description": "", "location": "", "places": [], "remote": None},
"fake:acme", "fake:acme")
item.source = "ats"
store.upsert_role(item, store.subscriptions(source="ats")[0])
again = store.role(one)
assert again is not None and again.description == "Python in full"
assert again.place == "Copenhagen"
def test_a_board_switched_off_since_the_plan_reads_nothing(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
sub = board(store, "fake:acme")
uid = role(store, "fake:acme", "1")
store.update_subscription(sub.id, fetch=0)
assert enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0
assert web["asked"] == []
# ------------------------------------------------------------------ cascade
class _Context:
def __init__(self) -> None:
self.made: list[Any] = []
def task(self, t: Any) -> Any:
self.made.append(t)
return t
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
context = _Context()
empty = Tally(results=({"filled": 0},))
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
context=context)
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
context=context)
assert out["next"] == "" and len(context.made) == 2
class _Embedder:
def encode(self, texts: list[str]) -> np.ndarray:
return np.vstack([np.asarray([t.lower().count("python"), 0.1], dtype=np.float32)
for t in texts])
def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
store: Store, tmp_path: Path, web: dict[str, Any],
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(embed, "load_embedder", lambda model: _Embedder())
board(store, "fake:acme")
one, two = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
store.put_vectors("fake", [(uid, np.ones(2, dtype=np.float32)) for uid in (one, two)])
agent = scored(store, "One", {one: 0.9, two: 0.1})
web["answers"][f"{BASE}/acme/1"] = b"Python in full"
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config(tmp_path), store,
workers={"cpu": 1, "http": 1, "llm": 1})
started = queue.enqueue("enrich")
queue.drain(timeout=60)
assert started is not None
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
key=lambda j: j.id)
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
assert web["asked"] == [f"{BASE}/acme/1"]
embedded = next(j for j in jobs if j.kind == "embed")
assert embedded.result["embedded"] == 1
assert set(store.vectors("fake")) == {one, two}
assert store.scores(agent.id)
# ----------------------------------------------------------------- triggers
@pytest.fixture
def app(tmp_path: Path) -> FastAPI:
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
@pytest.fixture
def client(app: FastAPI) -> Iterator[TestClient]:
with TestClient(app) as c:
yield c
ANY_STATE = ("queued", "running", "done", "failed", "cancelled")
def _enrich_behind_a_score(queue: Any) -> Any:
found = queue.jobs.last("enrich", states=ANY_STATE)
assert found is not None
assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"]
assert found.after == "done"
return found
def test_saving_an_agent_queues_a_score_then_enrich(client: TestClient, app: FastAPI) -> None:
res = client.post("/api/agents", json={"name": "Backend", "intent": "python backend"})
assert res.status_code == 200
_enrich_behind_a_score(app.state.queue)
res = client.post("/api/agents", json={"id": res.json()["id"], "name": "Backend",
"intent": "go backend"})
assert res.status_code == 200
def test_a_company_affinity_queues_a_score_then_enrich(
client: TestClient, app: FastAPI) -> None:
store: Store = app.state.store
store.save_agent("Backend", "python backend")
store.add_company("Acme")
res = client.post("/api/companies/acme/affinity", json={"affinity": 2})
assert res.status_code == 200
found = _enrich_behind_a_score(app.state.queue)
assert found.depends_on == [res.json()["score_job"]]
def test_one_score_is_one_enrich(app: FastAPI) -> None:
queue = app.state.queue
score = queue.enqueue("score")
first = cascade.enrich_after(queue, score)
assert cascade.enrich_after(queue, score).id == first.id
assert cascade.enrich_after(queue, None) is None
def test_a_vote_queues_neither_score_nor_enrich(client: TestClient, app: FastAPI) -> None:
store: Store = app.state.store
store.save_agent("Backend", "python backend")
item = Item(source_id="fake:acme:1", location=f"{BASE}/acme/1", title="Backend Engineer",
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme"})
item.source = "ats"
store.upsert_role(item)
assert client.post(f"/api/roles/{item.uid}/vote", json={"value": 1}).status_code == 200
assert client.delete(f"/api/roles/{item.uid}/vote").status_code == 200
queue = app.state.queue
assert queue.jobs.last("enrich", states=ANY_STATE) is None
assert queue.jobs.last("score", states=ANY_STATE) is None
+136 -163
View File
@@ -1,6 +1,7 @@
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
every host: the registry and its contract check, the fetch a host is handed,
the per-role detail reads, and Lever's EU instance. No network —
the per-role detail reads the `enrich` stage makes, and Lever's EU
instance. No network —
a fake host stands in where a behaviour belongs to no real host yet."""
from __future__ import annotations
@@ -11,7 +12,7 @@ import subprocess
import sys
import types
from collections.abc import Callable
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -308,34 +309,26 @@ def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
# ---------------------------------------------------------------- detail
def test_detail_is_read_for_unread_roles_only(
def test_a_pull_reads_no_role_page(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The pages are `enrich`'s: the pull reads the listing only, and sends
every row of a host with `detail` without a description."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
module = _fake_host(detail=_detail)
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
# What detail returns replaces the listing's field; a role not read keeps the listing's.
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert web["asked"] == [f"{base}/list"]
assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")]
assert "ats:detailed:fake:acme" not in state
assert state["ats:complete:fake:acme"] == "1"
def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
def test_a_role_read_on_its_page_keeps_the_place_its_page_gave(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The listing states a place on every pull, and a filled value replaces
a filled column, so for a role already read it is withheld."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full",
f"{base}/3": RuntimeError("down")})
module = _fake_host(detail=lambda get, slug, row: {
"description": get(f"{base}/{row['source_id']}").decode()})
web["answers"].update({f"{base}/list": b'["1", "3"]'})
module = _fake_host(detail=_detail)
listing = module.parse
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
for row in listing(body, slug)]
@@ -345,70 +338,25 @@ def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
assert by_id["fake:acme:1"] == ("", "", None)
# Read now, and its page stated neither: the listing's stand.
assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid")
# Never read: the listing's stand until its page is.
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
ids = [str(n) for n in range(5)]
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
module = _fake_host(detail=_detail)
items, state = _run(monkeypatch, module)
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
assert state["ats:complete:fake:acme"] == "1"
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": _status(429), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
assert state["ats:complete:fake:acme"] == "1"
def test_ids_no_longer_listed_leave_the_detailed_list(
def test_ids_no_longer_listed_leave_the_detail_lists(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""A listing cut short says nothing about the roles it left out, so their
ids stay, and the next pull does not read them in detail again."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
ids stay, and they are not read on their pages again."""
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
@@ -417,6 +365,19 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
assert state["ats:complete:fake:acme"] == ""
def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
roles: Any = source.fetch() # a generator, which `close` stops
assert next(roles).body == ""
roles.close()
assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
def test_fetch_board_reads_the_listing_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
@@ -435,63 +396,137 @@ def test_a_host_without_detail_keeps_the_listings_description(
assert "ats:detailed:fake:acme" not in state
def test_a_detail_answering_404_or_410_counts_as_read(
# -------------------------------------------------- read_board_details
NOW = datetime(2026, 9, 30, 12, tzinfo=UTC)
def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]:
"""`ats.read_board_details` over the fake host's roles `ids`, stored
without text, in that order."""
monkeypatch.setitem(readers.HOSTS, "fake", module)
rows = [row | {"description": ""}
for row in module.parse(json.dumps(ids).encode(), "acme")]
return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type]
now=kwargs.pop("now", NOW), **kwargs)
def test_details_are_read_in_the_order_given_for_unread_roles_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
f"{base}/3": b"three"}) # 1 answers 404
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [i.body for i in items] == ["", "", "three"]
web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full",
f"{base}/3": b"three in full"})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
"ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"])
assert web["asked"] == [f"{base}/3", f"{base}/1"]
assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [
("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
def test_details_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
ids = [str(n) for n in range(5)]
web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids})
module = _fake_host(detail=_detail)
state: dict[str, str] = {}
filled = _details(monkeypatch, module, state, ids, cap=2)
assert [r["description"] for r in filled] == ["full 0", "full 1"]
web["asked"].clear()
filled = _details(monkeypatch, module, state, ids[2:], cap=2)
assert web["asked"] == [f"{base}/2", f"{base}/3"]
def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"),
f"{base}/3": _status(429), f"{base}/4": b"four"})
state: dict[str, str] = {}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"])
assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"]
assert [r["source_id"] for r in filled] == ["1"]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
# A 429 is not the role's fault.
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
"""Two roles that always fail and a cap of two: the third is read on the
second pull, since roles never tried go before those that failed."""
"""Roles never tried go before those that failed, within the cap."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
web["answers"].update({f"{base}/1": failure, f"{base}/2": failure,
f"{base}/3": b"three", f"{base}/4": b"four"})
module = _fake_host(detail=_detail)
_items, state = _run(monkeypatch, module)
state: dict[str, str] = {}
_details(monkeypatch, module, state, ["1", "2"])
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
assert [i.body for i in items] == ["", "", "three", "four"]
_details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2)
assert web["asked"] == [f"{base}/3", f"{base}/4"]
# With nothing new left, the failed ones are tried again, within the cap.
web["asked"].clear()
web["answers"][f"{base}/1"] = b"one at last"
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one at last", "", "", ""]
filled = _details(monkeypatch, module, state, ["1", "2"], cap=2)
assert [r["description"] for r in filled] == ["one at last"]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
def test_a_detail_answering_404_or_410_counts_as_read(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404
state: dict[str, str] = {}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"])
assert [r["source_id"] for r in filled] == ["3"]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
assert json.loads(state["ats:detailed:fake:acme"]) == []
def test_failed_ids_no_longer_listed_leave_the_failed_list(
def test_a_read_without_text_is_read_again_after_some_days(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""Finding 18: an Oracle requisition not yet published has an empty page
today and a full one next week."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
web["answers"][f"{base}/1"] = b""
module = _fake_host(detail=lambda get, slug, row: {
"description": get(f"{base}/{row['source_id']}").decode()})
state: dict[str, str] = {}
assert _details(monkeypatch, module, state, ["1"]) == []
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
assert json.loads(state["ats:detailempty:fake:acme"]) == {
"fake:acme:1": NOW.isoformat(timespec="seconds")}
web["asked"].clear()
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1)
assert _details(monkeypatch, module, state, ["1"], now=later) == []
assert web["asked"] == []
web["answers"][f"{base}/1"] = b"published now"
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS)
[filled] = _details(monkeypatch, module, state, ["1"], now=later)
assert filled["description"] == "published now"
assert json.loads(state["ats:detailempty:fake:acme"]) == {}
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
def test_a_role_read_before_the_dates_were_kept_is_read_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"]["https://jobs.fake.example/acme/1"] = b"one"
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
[filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"])
assert filled["description"] == "one"
def test_detail_fills_only_the_fields_a_later_pull_keeps(
@@ -499,9 +534,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
caplog: pytest.LogCaptureFixture) -> None:
"""`salary_text`, `department` and `title` would be rewritten from the
listing on the next pull, so detail does not fill them."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]'})
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
"salary_min": 60000, "salary_max": 70000,
@@ -510,7 +542,8 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
"title": "Another title"}
caplog.set_level("DEBUG", logger=_runner.log.name)
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
[row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"])
item = ats.item_of(row, "fake:acme", "Acme")
assert (item.body, item.title, item.published_at) == (
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
@@ -522,66 +555,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
"salary_max", "posted"}
def _paged_host() -> types.ModuleType:
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
module: Any = _fake_host(detail=_detail)
del module.feed_url, module.parse
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
index = int(cursor or 0)
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
module.page = page
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
assert not readers.problems(module, "fake")
return module
def test_a_paged_host_reads_every_page_then_each_role_once(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
f"{base}/1": b"one", f"{base}/2": b"two"})
items, state = _run(monkeypatch, _paged_host())
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "two"]
assert state["ats:complete:fake:acme"] == "1"
web["asked"].clear()
items, state = _run(monkeypatch, _paged_host(), state)
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
assert [i.body for i in items] == ["", ""]
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
f"{base}/1": b"one"})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
_items, state = _run(monkeypatch, _paged_host(), state)
assert state["ats:complete:fake:acme"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The consumer stops after the first role, so the second, read but not
stored, is read again on the next pull."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
roles: Any = source.fetch() # a generator, which `close` stops
assert next(roles).body == "one"
roles.close()
assert "ats:detailed:fake:acme" not in source.state
# ----------------------------------------------------------------- Lever EU
def test_a_lever_slug_may_name_the_eu_instance() -> None:
+11 -9
View File
@@ -1053,7 +1053,9 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
assert batch is not None
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
assert stages == list(cascade.CASCADES["pull"])
# Enrich found no board with role pages, so the embed and score behind it
# are passed over.
assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"]
work = [j.kind for j in jobs if j.kind != FINAL]
# The `pull` job and the chain it asked for are one cascade, so the Jobs
# tab shows a scheduled pull as one run. Extract and summarise are off and
@@ -1107,10 +1109,10 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
"summarise")
assert [cascade.follows("pull", at) for at in range(6)] == \
["embed", "score", "extract", "score", "summarise", ""]
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score",
"extract", "score", "summarise")
assert [cascade.follows("pull", at) for at in range(9)] == \
["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
assert cascade.follows("nope", 0) == ""
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
@@ -1124,10 +1126,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
made.append(t)
return t
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5},
context=Context())
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 5
assert out["next"] == "summarise" and made[1].carries["at"] == 8