diff --git a/app/config.py b/app/config.py index b11835d..ebd3849 100644 --- a/app/config.py +++ b/app/config.py @@ -66,6 +66,25 @@ class SummariseConfig(ModelTaskConfig): batch_size: int = Field(10, ge=1) +class EnrichConfig(Section): + """Reading roles on their own pages after scoring, for the hosts whose + list carries no text. See app/jobs/enrich.py. A role is read when it has + no text and an enabled agent with a prompt ranks it ungated among its + best `top` and at `min_score` or above: the extract bar (`Limits`, + 500 and 0.55) with the score a notch lower, since a role ranked on its + title alone is the one whose score its text moves most.""" + + enabled: bool = True + top: int = Field(500, ge=0) + min_score: float = Field(0.5, ge=0, le=1) + #: Reads per board and per run. Per board is the cap the pull had; the + #: boards are read in parallel on the `http` pool. + per_board: int = Field(200, ge=0) + per_run: int = Field(1000, ge=0) + #: Days before a page that gave no text is read again. + retry_days: int = Field(7, ge=1) + + class Calibration(Section): """Where one embedding model's cosines sit, as `(lo, hi)` for fit and for taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a @@ -181,6 +200,7 @@ class Config(Section): extract: ExtractConfig = Field(default_factory=ExtractConfig) summarise: SummariseConfig = Field(default_factory=SummariseConfig) embed: EmbedConfig = Field(default_factory=EmbedConfig) + enrich: EnrichConfig = Field(default_factory=EnrichConfig) #: Starter agents, imported once into an empty table. agents: list[AgentSeed] = Field(default_factory=list) #: Starter searches, imported once into an empty set of `search` diff --git a/app/jobs/cascade.py b/app/jobs/cascade.py index 979eed6..f99863b 100644 --- a/app/jobs/cascade.py +++ b/app/jobs/cascade.py @@ -1,6 +1,6 @@ """The tasks roledesk offers, and the order a pull runs them in. - pull → embed → score → extract → score → summarise + pull → embed → score → enrich → embed → score → extract → score → summarise newsdesk's cascade convention: a cascade is a tuple of task names, and the final job of one task starts the next (`step` is the callback it carries). @@ -11,7 +11,10 @@ Every stage is always in the chain, and a stage that is switched off finds nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not satisfy, is an extract stage with no batches whose final job moves straight on. The free stages go first: embed and score cost no tokens, and the score -they produce is what bounds the two that do. Extract reads only the best +they produce is what bounds the rest. Enrich reads the best roles without +text on their own pages (`app/jobs/enrich.py`), and embed and score follow for +the roles it filled; when it filled none, `step` passes over both. Extract +reads only the best roles of each enabled agent, as deep and as low as that agent's own `limits` say; the second score then applies the gates to the columns it filled. Summarise runs last, on the best roles by the same agent's summarise limits. @@ -23,6 +26,11 @@ having either way. `pull` is a job as well as a cascade, because a schedule enqueues a job kind and a cascade is started by one: the `pull` job's only work is to ask for the chain. Its default schedule, `every 6h`, is declared here beside it. + +`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued +behind the `score` that follows a change to the ranking other than a pull's +(`enrich_after`): an agent saved, a company's affinity changed. A vote moves +one agent's taste a little and queues neither. """ from __future__ import annotations @@ -32,16 +40,21 @@ from dataclasses import replace from functools import partial from typing import Any -from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs +from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs from desk.tools import agent from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job STEP = "app.jobs.cascade:step" CASCADES: dict[str, tuple[str, ...]] = { - "pull": ("pull", "embed", "score", "extract", "score", "summarise"), + "pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score", + "summarise"), + "enrich": ("enrich", "embed", "score"), } +#: What follows `enrich` for the roles it filled, passed over when it filled none. +REFILL = ("embed", "score") + def _once(config: Any, store: Any) -> list[dict[str, Any]]: """One batch with nothing in its payload: a task that is a single job.""" @@ -91,6 +104,7 @@ STAGES: dict[str, Callable[[str], Task]] = { "score": lambda cascade: Task(kind="score", finder=_once, label="Scoring roles for every agent"), "summarise": _summarise, + "enrich": lambda cascade: enrich.enrich_task(), } @@ -115,6 +129,10 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *, chain goes on, as in newsdesk's cascade.""" name, done = payload.get("cascade", ""), payload.get("task", "") at = int(payload.get("at", -1)) + tasks = CASCADES.get(name, ()) + if done == "enrich" and not enrich.filled(tally) \ + and tasks[at + 1:at + 1 + len(REFILL)] == REFILL: + at += len(REFILL) following = follows(name, at) if following: context.task(task(following, cascade=name, at=at + 1)) @@ -169,6 +187,26 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str return {"started": " > ".join(CASCADES["pull"]), "run": run} +@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http") +def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: + """Ask for the enrich cascade, `enrich → embed → score`. Its own work is + nil, as the `pull` job's. Under `NULL` there is no queue to take it.""" + if not context.task(chain("enrich")): + raise RuntimeError("no queue to start the enrich cascade on") + return {"started": " > ".join(CASCADES["enrich"])} + + +def enrich_after(queue: Any, score_job: Any) -> Any: + """Queue the `enrich` job behind `score_job`, the rescore that follows a + change to the ranking, however that score ends. One per score job + (`enrich:after:`), so ten changes answered by one score are one + enrich. None without a score job, or when the queue refuses.""" + if score_job is None: + return None + return queue.enqueue("enrich", depends_on=[score_job.id], after="done", + dedup_key=f"enrich:after:{score_job.id}") + + #: What `app job ` and `/api/jobs` start besides `pull`: one stage #: each, on its own. `pull` is not a task here, since it is a job: a name that #: is both would be listed twice by `/api/jobs/kinds`. @@ -182,4 +220,5 @@ for _name in ("extract", "embed", "score", "summarise"): # setting: a cadence edited or switched off under Runs survives # every restart (see pipeline/schedule.py). declare("pull", "pull", "every 6h", - label="Pull every board, then embed, score, extract, rescore and summarise") + label="Pull every board, then embed, score, read the best roles' pages, " + "extract, rescore and summarise") diff --git a/app/jobs/enrich.py b/app/jobs/enrich.py new file mode 100644 index 0000000..f6f98e9 --- /dev/null +++ b/app/jobs/enrich.py @@ -0,0 +1,168 @@ +"""`enrich`: the best roles without text, read on their own pages. + +Some hosts list a role without its text and state it on the role's own page +only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest +with a `detail`). The pull reads listings only; this stage reads those pages +after scoring, so the calls go where the ranking is: + +1. `enrich_waiting` hands over the open roles without text that an enabled + agent with a prompt ranks ungated among its best `enrich.top` and at + `enrich.min_score` or above, best first by the highest such score. A role + below that bar for every agent gets no call. +2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most + `enrich.per_board` from one board and `enrich.per_run` in all, leaving + out the roles already read, unless their read failed or gave no text + `enrich.retry_days` ago (`plugins.ats.detail_due`). +3. One batch per board, on the `http` pool, reads them + (`plugins.ats.read_board_details`, which keeps `ats:detailed:`, + `ats:detailfailed:` and `ats:detailempty:`) and stores + what each page filled. `Store.upsert_role` drops the vector of a role + whose text improved, so the next `embed` encodes those roles only. + +In the pull cascade `embed` and `score` follow; `cascade.step` passes over +both when no role was filled. The same stage follows a rescore after an agent +is saved or a company's affinity changes (`cascade.enrich_after`), never after +a vote. Whether it runs is `stages.stage_on(config, store, "enrich")`. +""" + +from __future__ import annotations + +import json +import logging +from collections import defaultdict +from datetime import datetime +from typing import Any + +import httpx + +from app.config import Config +from app.jobs import stages +from app.storage import Store +from pipeline import NULL, Batch, RunContext, Tally, Task, job + +log = logging.getLogger(__name__) + +KIND = "enrich_batch" +SETTLE = "app.jobs.enrich:settle" + + +def _ats() -> Any: + """`plugins.ats`, imported when needed, the way `pull.run_batch` does.""" + from plugins import ats + + return ats + + +def _row(found: dict[str, Any], target: str) -> dict[str, Any]: + """A stored role back as the row its reader listed: the stored `raw` + with the host's own id, the title, the link and the date.""" + try: + raw = json.loads(found.get("raw") or "{}") + except ValueError: + raw = {} + raw = raw if isinstance(raw, dict) else {} + try: + posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or "")) + except ValueError: + posted = None + return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"), + "title": found["title"], "url": found["url"], "description": "", + "posted": posted} + + +def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]: + """`(target, uids)` per board, best board first, within the budgets: the + roles `enrich_waiting` ranks, on fetchable boards whose reader has + `detail` and that are not failing, and due for a read.""" + settings = config.enrich + ats = _ats() + boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)} + by_board: dict[str, list[dict[str, Any]]] = defaultdict(list) + for found in store.enrich_waiting(settings.top, settings.min_score): + target = boards.get(found["subscription_id"]) + if target is None: + continue + module = ats.readers.HOSTS.get(target.partition(":")[0]) + if module is None or not hasattr(module, "detail") \ + or f"ats:failing:{target}" in store.state: + continue + by_board[target].append(found) + due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found], + retry_days=settings.retry_days)) + for target, found in by_board.items()} + # Best first; roles whose read failed after every other, so a page that + # keeps failing cannot take the budget of the roles never tried. + failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1]) + for target in by_board} + ranked = sorted((f for found in by_board.values() for f in found), + key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]], + -float(f["score"] or 0), f["uid"])) + picked: dict[str, list[str]] = {} + total = 0 + for found in ranked: + target = boards[found["subscription_id"]] + if found["source_id"] not in due[target]: + continue + if total >= settings.per_run: + break + uids = picked.setdefault(target, []) + if len(uids) >= settings.per_board: + continue + uids.append(found["uid"]) + total += 1 + return list(picked.items()) + + +def batches(config: Config, store: Store) -> list[Batch]: + """The finder: one batch per board, nothing while the stage is off.""" + if not stages.stage_on(config, store, "enrich"): + return [] + return [Batch({"target": target, "uids": uids}, + label=f"Reading {len(uids)} roles on {target}") + for target, uids in plan(config, store)] + + +@job(KIND, "Reading roles on their own pages", resource="http") +def run_batch(config: Config, store: Store, *, target: str, uids: list[str], + context: RunContext = NULL) -> dict[str, Any]: + """Read one board's roles in `uids` on their own pages, in that order, + and store what each page filled. `filled` is how many roles changed, + `texts` how many of them now have text.""" + ats = _ats() + subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target] + if not subs or not uids: + return {"target": target, "filled": 0, "texts": 0} + sub = subs[0] + found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)} + rows = [_row(found[uid], target) for uid in uids if uid in found] + context.check() + with httpx.Client() as client: + filled = ats.read_board_details(store.state, target, rows, client=client, + cap=len(rows), retry_days=config.enrich.retry_days) + label = sub.company or sub.label + texts = 0 + for row in filled: + item = ats.item_of(row, target, label) + item.source = "ats" + store.upsert_role(item, sub) + texts += bool(row.get("description")) + log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled), + texts) + return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts} + + +def filled(tally: Tally) -> int: + """How many roles the batches behind `tally` filled.""" + return sum(int(result.get("filled") or 0) for result in tally.results) + + +def settle(config: Config, store: Store, tally: Tally, *, + context: RunContext = NULL) -> dict[str, Any]: + """The run's sum: roles filled, and how many of them now have text.""" + return {"filled": filled(tally), + "texts": sum(int(result.get("texts") or 0) for result in tally.results)} + + +def enrich_task() -> Task: + return Task(kind=KIND, finder=batches, name="enrich", settle=SETTLE, + label="Reading the best roles on their own pages") diff --git a/app/jobs/stages.py b/app/jobs/stages.py index b121a50..7b89de9 100644 --- a/app/jobs/stages.py +++ b/app/jobs/stages.py @@ -1,4 +1,5 @@ -"""Whether the two model stages of a pull, `extract` and `summarise`, run. +"""Whether the two model stages of a pull, `extract` and `summarise`, and +the `enrich` stage run. `extract.enabled` and `summarise.enabled` in config.yaml are read at startup and say nothing on the page, so the switch lives in the database as well: the @@ -8,9 +9,13 @@ default for a fresh database and a deployment that never uses the switch behaves as it did. `stage_on` is the one place that decides, and both tasks' finders ask it (`extract.extract_task`, `summarise.summarise_task`). -`describe` is what `GET /api/stages` and `app stages` show: for each stage -whether it is on, where that comes from, the model it uses, and who would -answer. +`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`) +asks no model and is on by default (`enrich.enabled`); its switch is +`stage:enrich`, set the same way through `PUT /api/stages/enrich`. + +`describe` is what `GET /api/stages` and `app stages` show: for each model +stage whether it is on, where that comes from, the model it uses, and who +would answer. """ from __future__ import annotations @@ -25,6 +30,9 @@ from desk.tools import agent NAMES = ("extract", "summarise") +#: Every stage with a switch: the model stages and `enrich`. +SWITCHES = (*NAMES, "enrich") + NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, " "see docs/llm-backends.md") ANSWERS = { @@ -39,7 +47,7 @@ class Stage(BaseModel): `backend` is `agent.backend`'s answer, "" for nobody, and `answer` says it in words.""" - name: Literal["extract", "summarise"] + name: Literal["extract", "summarise", "enrich"] on: bool source: Literal["stored", "config"] model: str @@ -65,7 +73,7 @@ def stage_on(config: Config, store: Store, name: str) -> bool: def set_stage(store: Store, name: str, on: bool) -> None: """Store the switch. A `KeyError` for a name that is not a stage.""" - if name not in NAMES: + if name not in SWITCHES: raise KeyError(name) store.state[_key(name)] = "on" if on else "off" @@ -84,7 +92,8 @@ def describe_one(config: Config, store: Store, name: str) -> Stage: backend, said = answer(config) return Stage(name=name, on=stage_on(config, store, name), source="config" if stored(store, name) is None else "stored", - model=getattr(config, name).model, backend=backend, answer=said) + model=getattr(getattr(config, name), "model", ""), backend=backend, + answer=said) def describe(config: Config, store: Store) -> list[Stage]: diff --git a/app/storage/store.py b/app/storage/store.py index 4f8d184..180d8da 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -1398,6 +1398,46 @@ class Store: return None return self.role(uid) + # --------------------------------------------------------------- enrich + + def enrich_waiting(self, top: int, floor: float, + uids: Sequence[str] | None = None) -> list[dict[str, Any]]: + """Open `ats` roles without text that an enabled agent with a prompt + ranks ungated among its best `top` and at `floor` or above, best + first by the highest such score, as `{uid, source_id, + subscription_id, title, url, published_at, raw, score}`. The ranking + is `_top_scope`'s, with the bar given rather than the agent's own. + Given `uids`, those of them still without text, whatever their rank, + in no order (`score` is None), since a batch finishes what it was + dealt.""" + fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, " + "r.published_at, r.raw") + waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source = 'ats' " + "AND r.subscription_id IS NOT NULL AND r.description = '' " + "AND r.article_text = ''") + if uids is not None: + rows = self._rows( + f"SELECT {fields}, NULL AS score FROM roles r WHERE {waiting} " + "AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),)) + else: + rows = self._rows( + f"""SELECT {fields}, MAX(ranked.score) AS score FROM ( + SELECT s.uid, s.score, ROW_NUMBER() OVER ( + PARTITION BY s.agent_id + ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n + FROM scores s + JOIN agents a ON a.id = s.agent_id AND a.enabled = 1 + AND EXISTS (SELECT 1 FROM agent_prompts p + WHERE p.agent_id = a.id + AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '') + JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL + AND top.dup_of IS NULL + WHERE s.gated_reason = '') ranked + JOIN roles r ON r.uid = ranked.uid + WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting} + GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor)) + return [dict(row) for row in rows] + # -------------------------------------------------------------- vectors def roles_to_embed(self, model: str, limit: int) -> list[Role]: diff --git a/app/web/api.py b/app/web/api.py index 4afc9fd..c0e8a85 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -679,6 +679,7 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, except KeyError: raise HTTPException(404, f"no such company: {key}") from None job = _score_after_change(queue) + jobs.cascade.enrich_after(queue, job) return AffinityOut(**company.model_dump(), score_job=job.id if job else None) @@ -963,17 +964,19 @@ class AgentBody(BaseModel): @router.post("/agents") -def save_agent(body: AgentBody, store: StoreDep) -> Agent: - """Scores are not recomputed here: the page asks `POST /api/score` after - a save and waits on the job it answers with.""" +def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent: + """Queues the rescore, which the page's `POST /api/score` after a save + joins, and the enrich run behind it (`cascade.enrich_after`).""" try: - return store.save_agent(body.name, body.intent, body.gates, body.weights, - body.enabled, agent_id=body.id, limits=body.limits, - prompts=body.prompts) + saved = store.save_agent(body.name, body.intent, body.gates, body.weights, + body.enabled, agent_id=body.id, limits=body.limits, + prompts=body.prompts) except ValueError as exc: raise HTTPException(409, str(exc)) from exc except KeyError: raise HTTPException(404, f"no such agent: {body.id}") from None + jobs.cascade.enrich_after(queue, _score_after_change(queue)) + return saved @router.delete("/agents/{agent_id}", status_code=204) diff --git a/plugins/ats.py b/plugins/ats.py index 167835d..5b718dc 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -20,6 +20,10 @@ recorded as failing under `ats:failing:`, and skipped. A board's subscription `params` (`{"countries": ["DK"], "remote": true}`) are its filter, narrowed to what its reader declares (`_runner.ask_for`). +A pull reads listings only. A host that states a role's text on the role's +own page (`text="detail"`) is read there by `app/jobs/enrich.py` after +scoring, best roles first, through `read_board_details`. + The limits below are read when a board is read, so a test that replaces one on this module reaches the runner. """ @@ -31,7 +35,7 @@ import logging import re import sys from collections.abc import Callable, Iterator -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -363,41 +367,152 @@ class ATSSource(Source): # Matches `roles.source_id`, which `Store.mark_closed` compares. return f"{host}:{slug}:{source_id}" + # A host with `detail` states a role's text on the role's own page, + # which the `enrich` stage reads after scoring (`read_board_details`), + # never the pull. Its rows go out without a description, so the text + # a role's page gave stays, and a role read on its own page goes out + # without the listing's place and remote as well, so what its page + # said stays too (see `_runner.read_details`). rows = listing.rows - detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}") detail_sets: tuple[set[str], set[str]] | None = None if hasattr(module, "detail"): - detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys) - listed = {compound(i) for i in listing.ids} if listing.complete else None - rows, detailed, failed = _runner.read_details( - module, host, slug, get, rows, detailed, failed, listed=listed, - limits=_limits()) - detail_sets = (detailed, failed) + detailed, failed = (_runner.id_set(self.state, key) + for key in detail_keys(target)[:2]) + rows = [row | {"description": ""} + | (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {}) + for row in rows] + if listing.complete: + listed = {compound(i) for i in listing.ids} + detail_sets = (detailed & listed, failed & listed) for row in rows: - description = row.get("description") or "" - yield Item( - source_id=compound(row["source_id"]), - location=row["url"], - title=row["title"], - excerpt=description[:300], - body=description, - published_at=row.get("posted"), - subscription=label, - kind="role", - raw=_runner.raw_of(row) | { - "company": label, - "function": classify(row.get("department") or "", row["title"]), - "board": host, - "slug": slug, - }, - ) + yield item_of(row, target, label) # Stamped only once every row went out, so a board that blew up half # way through is neither marked ok nor has a partial id list treated - # as the whole of it, and a pull stopped half way reads the rest of - # its details again. + # as the whole of it. _runner.stamp(self.state, "ats", target, run_marker, listing, [compound(i) for i in listing.ids]) if detail_sets is not None: - for key, ids in zip(detail_keys, detail_sets, strict=True): - self.state[key] = json.dumps(sorted(ids)) + # A complete listing drops the ids it no longer lists. + for key, ids in zip(detail_keys(target), detail_sets, strict=False): + if key in self.state: + self.state[key] = json.dumps(sorted(ids)) + + +# ------------------------------------------------------ the per-role reads + +#: What a pull withholds for a role already read on its own page: the +#: listing's place and remote, which would replace what the page stated. +_KEPT_FROM_DETAIL: dict[str, Any] = {"location": "", "places": [], "remote": None} + +#: Days before a role whose own page gave no text is read again: an Oracle +#: requisition not yet published, an empty SuccessFactors block. A 404 or a +#: 410 counts the same, since the listing closes a role that is gone. +EMPTY_RETRY_DAYS = 7 + + +def detail_keys(target: str) -> tuple[str, str, str]: + """The state keys of one board's per-role reads: the ids read, the ids + whose read failed, and `{id: date}` of the reads that gave no text.""" + return (f"ats:detailed:{target}", f"ats:detailfailed:{target}", + f"ats:detailempty:{target}") + + +def _dates(state: Any, key: str) -> dict[str, str]: + try: + value = json.loads(state.get(key) or "{}") + except (TypeError, ValueError): + return {} + return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {} + + +def detail_due(state: Any, target: str, ids: list[str], *, now: datetime | None = None, + retry_days: int = EMPTY_RETRY_DAYS) -> list[str]: + """Of `ids` (compound, `host:slug:id`, roles without text), those to read + on their own page, in the order given: never read, failed, or read with + no text at least `retry_days` ago. A role read before its date was kept + counts as read that long ago.""" + detailed = _runner.id_set(state, detail_keys(target)[0]) + empty = _dates(state, detail_keys(target)[2]) + cutoff = ((now or datetime.now(UTC)) - timedelta(days=retry_days)).isoformat( + timespec="seconds") + return [i for i in ids if i not in detailed or empty.get(i, "") <= cutoff] + + +def read_board_details(state: Any, target: str, rows: list[Row], *, client: httpx.Client, + cap: int | None = None, now: datetime | None = None, + retry_days: int = EMPTY_RETRY_DAYS) -> list[Row]: + """Read `rows` of one board on their own pages, in the order given with + the roles whose read failed last, at most `cap` (`DETAIL_CAP`), and keep the board's per-role state + (`detail_keys`). The rows are the stored roles without text, each with + its host's own `source_id`. Returns the rows whose page filled a field, + each merged with what it filled. + + A read that gave no text is kept with its date and read again once it is + `retry_days` old (`detail_due`). The sets are read again just before they + are written and changed by what this read did, so a pull that dropped ids + meanwhile keeps its drop.""" + host, _, slug = target.partition(":") + module = readers.HOSTS[host] + keys = detail_keys(target) + now = now or datetime.now(UTC) + + def compound(row: Row) -> str: + return f"{host}:{slug}:{row['source_id']}" + + due = set(detail_due(state, target, [compound(r) for r in rows], now=now, + retry_days=retry_days)) + detailed, failed = (_runner.id_set(state, key) for key in keys[:2]) + # Roles whose read failed go after the rest, so they cannot starve them. + wanted = sorted((row for row in rows if compound(row) in due), + key=lambda row: compound(row) in failed) + wanted = wanted[:DETAIL_CAP if cap is None else cap] + if not wanted: + return [] + again = {compound(row) for row in wanted} & detailed + limits = _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=len(wanted)) + out, done, failing = _runner.read_details( + module, host, slug, _getter(client, host, slug), wanted, detailed - again, failed, + limits=limits) + read = done - (detailed - again) + filled: list[Row] = [] + empty = _dates(state, keys[2]) + for before, after in zip(wanted, out, strict=True): + key = compound(before) + if key in read: + if after.get("description"): + empty.pop(key, None) + else: + empty[key] = now.isoformat(timespec="seconds") + if after != before | {"description": ""}: + filled.append(after) + + detailed_now = (_runner.id_set(state, keys[0]) - again) | read + failed_now = (_runner.id_set(state, keys[1]) | (failing - failed)) - read + state[keys[0]] = json.dumps(sorted(detailed_now)) + state[keys[1]] = json.dumps(sorted(failed_now)) + state[keys[2]] = json.dumps({k: v for k, v in sorted(empty.items()) if k in detailed_now}) + return filled + + +def item_of(row: Row, target: str, label: str) -> Item: + """One row of board `target` as the `Item` the store takes: its id + compound, `host:slug:id`, as `roles.source_id` has it.""" + host, _, slug = target.partition(":") + description = row.get("description") or "" + return Item( + source_id=f"{host}:{slug}:{row['source_id']}", + location=row["url"], + title=row["title"], + excerpt=description[:300], + body=description, + published_at=row.get("posted"), + subscription=label, + kind="role", + raw=_runner.raw_of(row) | { + "company": label, + "function": classify(row.get("department") or "", row["title"]), + "board": host, + "slug": slug, + }, + ) diff --git a/tests/test_enrich.py b/tests/test_enrich.py new file mode 100644 index 0000000..541b95d --- /dev/null +++ b/tests/test_enrich.py @@ -0,0 +1,368 @@ +"""The `enrich` stage: the best roles without text, read on their own pages +after scoring. Which roles and in what order, the budgets, what is stored, +the cascade around it, and what triggers it. No network: a fake host stands +in, answering from a dict.""" + +from __future__ import annotations + +import json +import sys +import types +from collections.abc import Iterator +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import httpx +import numpy as np +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.config import Config, ServerConfig # noqa: E402 +from app.jobs import cascade, embed, enrich, stages # noqa: E402 +from app.main import create_app # noqa: E402 +from app.storage import Store # noqa: E402 +from desk.models import Item # noqa: E402 +from pipeline import FINAL, JobQueue, Jobs, Tally # noqa: E402 +from plugins import ats, readers # noqa: E402 + +BASE = "https://jobs.fake.example" + + +def _fake_host(name: str = "fake", *, detail: bool = True) -> types.ModuleType: + """A host whose role pages are `BASE//`, answered by `web`.""" + module: Any = types.ModuleType(name) + module.NAME, module.LABEL, module.PATTERNS = name, name.title(), () + module.slug_ok = lambda slug: slug.isalnum() + module.allowed = lambda slug: lambda h: h == "jobs.fake.example" + module.careers_url = lambda slug: f"{BASE}/{slug}" + module.feed_url = lambda slug: f"{BASE}/{slug}/list" + module.parse = lambda body, slug: [] + if detail: + module.detail = lambda get, slug, row: { + "description": get(f"{BASE}/{slug}/{row['source_id']}").decode(), + "location": "Copenhagen"} + return module + + +@pytest.fixture +def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """`ats._get` answering from `answers` (URL -> body), 404 otherwise, + remembering what it was asked; hosts `fake` and `plain` (no detail).""" + answers: dict[str, Any] = {} + asked: list[str] = [] + + def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None): + asked.append(url) + answer = answers.get(url) + if answer is None: + request = httpx.Request("GET", url) + raise httpx.HTTPStatusError("404", request=request, + response=httpx.Response(404, request=request)) + return answer + + monkeypatch.setattr(ats, "_get", fake_get) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + monkeypatch.setitem(readers.HOSTS, "fake", _fake_host()) + monkeypatch.setitem(readers.HOSTS, "plain", _fake_host("plain", detail=False)) + return {"answers": answers, "asked": asked} + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +def config(tmp_path: Path, **enrich_settings: Any) -> Config: + return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"}, + enrich=enrich_settings) + + +def board(store: Store, target: str) -> Any: + host = target.partition(":")[0] + return store.add_subscription("ats", target, label=target, company=target, board=host) + + +def role(store: Store, target: str, native: str, *, body: str = "", + title: str = "Backend Engineer") -> str: + """A role of board `target` as a pull stores it: without text on a host + with `detail`.""" + sub = next(s for s in store.subscriptions(source="ats") if s.target == target) + host, _, slug = target.partition(":") + item = Item(source_id=f"{target}:{native}", location=f"{BASE}/{slug}/{native}", + title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC), + raw={"company": target, "place": "2 Locations", "board": host, "slug": slug}) + item.source = "ats" + store.upsert_role(item, sub) + return item.uid + + +def scored(store: Store, name: str, scores: dict[str, float], *, + gated: tuple[str, ...] = ()) -> Any: + agent = store.save_agent(name, f"{name} python backend") + store.replace_scores(agent.id, [(uid, value, "gated" if uid in gated else "") + for uid, value in scores.items()]) + return agent + + +# -------------------------------------------------------------- which roles + +def test_roles_go_best_first_by_their_best_agent_score( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + board(store, "fake:acme") + board(store, "fake:beta") + a1, a2 = role(store, "fake:acme", "1"), role(store, "fake:acme", "2") + b1 = role(store, "fake:beta", "1") + scored(store, "One", {a1: 0.6, a2: 0.9, b1: 0.55}) + scored(store, "Two", {a1: 0.95, b1: 0.7}) + + assert [(r["uid"], r["score"]) for r in store.enrich_waiting(500, 0.5)] == [ + (a1, 0.95), (a2, 0.9), (b1, 0.7)] + assert enrich.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]), + ("fake:beta", [b1])] + + +def test_a_role_no_agent_ranks_high_enough_gets_no_call( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + board(store, "fake:acme") + low, gated, deep, kept = (role(store, "fake:acme", n) for n in ("1", "2", "3", "4")) + scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,)) + # `top` 1 per agent: only the best ungated role of each agent is in reach. + assert enrich.plan(config(tmp_path, top=1), store) == [("fake:acme", [kept])] + assert enrich.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])] + # A disabled agent ranks nothing. + agent = store.agents()[0] + store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, enabled=False, + agent_id=agent.id, limits=agent.limits, prompts=agent.prompts) + assert enrich.plan(config(tmp_path), store) == [] + + +def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + for target in ("fake:acme", "plain:acme", "fake:down"): + board(store, target) + wanted = role(store, "fake:acme", "1") + has_text = role(store, "fake:acme", "2", body="Python in full") + plain = role(store, "plain:acme", "1") + down = role(store, "fake:down", "1") + store.state["ats:failing:fake:down"] = json.dumps({"since": "2026-09-29", "reason": "500"}) + scored(store, "One", {wanted: 0.9, has_text: 0.95, plain: 0.9, down: 0.9}) + assert enrich.plan(config(tmp_path), store) == [("fake:acme", [wanted])] + + +def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + board(store, "fake:acme") + read, failed, fresh = (role(store, "fake:acme", n) for n in ("1", "2", "3")) + now = datetime.now(UTC).isoformat(timespec="seconds") + store.state["ats:detailed:fake:acme"] = json.dumps(["fake:acme:1"]) + store.state["ats:detailempty:fake:acme"] = json.dumps({"fake:acme:1": now}) + store.state["ats:detailfailed:fake:acme"] = json.dumps(["fake:acme:2"]) + scored(store, "One", {read: 0.9, failed: 0.8, fresh: 0.6}) + assert enrich.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])] + + +# ------------------------------------------------------------------ budgets + +def test_the_budgets_per_board_and_per_run( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + board(store, "fake:acme") + board(store, "fake:beta") + acme = [role(store, "fake:acme", str(n)) for n in range(4)] + beta = [role(store, "fake:beta", str(n)) for n in range(4)] + # Interleaved: acme 0.99, beta 0.98, acme 0.97, ... + scored(store, "One", {uid: 0.99 - 0.01 * n + for n, uid in enumerate(u for pair in zip(acme, beta, strict=True) + for u in pair)}) + + assert enrich.plan(config(tmp_path, per_board=2), store) == [ + ("fake:acme", acme[:2]), ("fake:beta", beta[:2])] + assert enrich.plan(config(tmp_path, per_run=3), store) == [ + ("fake:acme", acme[:2]), ("fake:beta", beta[:1])] + assert enrich.plan(config(tmp_path, per_board=1, per_run=5), store) == [ + ("fake:acme", acme[:1]), ("fake:beta", beta[:1])] + + +def test_the_stage_off_finds_nothing( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + board(store, "fake:acme") + scored(store, "One", {role(store, "fake:acme", "1"): 0.9}) + assert len(enrich.batches(config(tmp_path), store)) == 1 + assert enrich.batches(config(tmp_path, enabled=False), store) == [] + stages.set_stage(store, "enrich", True) + assert len(enrich.batches(config(tmp_path, enabled=False), store)) == 1 + stages.set_stage(store, "enrich", False) + assert enrich.batches(config(tmp_path), store) == [] + + +# ------------------------------------------------------------ what is stored + +def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + board(store, "fake:acme") + one, two, gone = (role(store, "fake:acme", n) for n in ("1", "2", "3")) + store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32)) for uid in (one, two, gone)]) + web["answers"].update({f"{BASE}/acme/1": b"Python in full", f"{BASE}/acme/2": b"Go too"}) + + out = enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one]) + + assert out == {"target": "fake:acme", "read": 3, "filled": 2, "texts": 2} + assert web["asked"] == [f"{BASE}/acme/2", f"{BASE}/acme/3", f"{BASE}/acme/1"] + first = store.role(one) + assert first is not None and first.description == "Python in full" + assert first.place == "Copenhagen" and first.title == "Backend Engineer" + assert set(store.vectors("fake")) == {gone} + assert json.loads(store.state["ats:detailed:fake:acme"]) == [ + "fake:acme:1", "fake:acme:2", "fake:acme:3"] + # A later pull of the same listing keeps the text and the page's place. + item = ats.item_of({"source_id": "1", "title": "Backend Engineer", "url": f"{BASE}/acme/1", + "description": "", "location": "", "places": [], "remote": None}, + "fake:acme", "fake:acme") + item.source = "ats" + store.upsert_role(item, store.subscriptions(source="ats")[0]) + again = store.role(one) + assert again is not None and again.description == "Python in full" + assert again.place == "Copenhagen" + + +def test_a_board_switched_off_since_the_plan_reads_nothing( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + sub = board(store, "fake:acme") + uid = role(store, "fake:acme", "1") + store.update_subscription(sub.id, fetch=0) + assert enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0 + assert web["asked"] == [] + + +# ------------------------------------------------------------------ cascade + +class _Context: + def __init__(self) -> None: + self.made: list[Any] = [] + + def task(self, t: Any) -> Any: + self.made.append(t) + return t + + +def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None: + context = _Context() + empty = Tally(results=({"filled": 0},)) + out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3}, + context=context) + assert out["next"] == "extract" and context.made[0].carries["at"] == 6 + + out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})), + {"cascade": "pull", "task": "enrich", "at": 3}, context=context) + assert out["next"] == "embed" and context.made[1].carries["at"] == 4 + + out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0}, + context=context) + assert out["next"] == "" and len(context.made) == 2 + + +class _Embedder: + def encode(self, texts: list[str]) -> np.ndarray: + return np.vstack([np.asarray([t.lower().count("python"), 0.1], dtype=np.float32) + for t in texts]) + + +def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles( + store: Store, tmp_path: Path, web: dict[str, Any], + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(embed, "load_embedder", lambda model: _Embedder()) + board(store, "fake:acme") + one, two = role(store, "fake:acme", "1"), role(store, "fake:acme", "2") + store.put_vectors("fake", [(uid, np.ones(2, dtype=np.float32)) for uid in (one, two)]) + agent = scored(store, "One", {one: 0.9, two: 0.1}) + web["answers"][f"{BASE}/acme/1"] = b"Python in full" + queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config(tmp_path), store, + workers={"cpu": 1, "http": 1, "llm": 1}) + + started = queue.enqueue("enrich") + queue.drain(timeout=60) + + assert started is not None + jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr] + key=lambda j: j.id) + assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"] + assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"] + assert web["asked"] == [f"{BASE}/acme/1"] + embedded = next(j for j in jobs if j.kind == "embed") + assert embedded.result["embedded"] == 1 + assert set(store.vectors("fake")) == {one, two} + assert store.scores(agent.id) + + +# ----------------------------------------------------------------- triggers + +@pytest.fixture +def app(tmp_path: Path) -> FastAPI: + return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), + database=tmp_path / "roledesk.db")) + + +@pytest.fixture +def client(app: FastAPI) -> Iterator[TestClient]: + with TestClient(app) as c: + yield c + + +ANY_STATE = ("queued", "running", "done", "failed", "cancelled") + + +def _enrich_behind_a_score(queue: Any) -> Any: + found = queue.jobs.last("enrich", states=ANY_STATE) + assert found is not None + assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"] + assert found.after == "done" + return found + + +def test_saving_an_agent_queues_a_score_then_enrich(client: TestClient, app: FastAPI) -> None: + res = client.post("/api/agents", json={"name": "Backend", "intent": "python backend"}) + assert res.status_code == 200 + _enrich_behind_a_score(app.state.queue) + res = client.post("/api/agents", json={"id": res.json()["id"], "name": "Backend", + "intent": "go backend"}) + assert res.status_code == 200 + + +def test_a_company_affinity_queues_a_score_then_enrich( + client: TestClient, app: FastAPI) -> None: + store: Store = app.state.store + store.save_agent("Backend", "python backend") + store.add_company("Acme") + res = client.post("/api/companies/acme/affinity", json={"affinity": 2}) + assert res.status_code == 200 + found = _enrich_behind_a_score(app.state.queue) + assert found.depends_on == [res.json()["score_job"]] + + +def test_one_score_is_one_enrich(app: FastAPI) -> None: + queue = app.state.queue + score = queue.enqueue("score") + first = cascade.enrich_after(queue, score) + assert cascade.enrich_after(queue, score).id == first.id + assert cascade.enrich_after(queue, None) is None + + +def test_a_vote_queues_neither_score_nor_enrich(client: TestClient, app: FastAPI) -> None: + store: Store = app.state.store + store.save_agent("Backend", "python backend") + item = Item(source_id="fake:acme:1", location=f"{BASE}/acme/1", title="Backend Engineer", + body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme"}) + item.source = "ats" + store.upsert_role(item) + + assert client.post(f"/api/roles/{item.uid}/vote", json={"value": 1}).status_code == 200 + assert client.delete(f"/api/roles/{item.uid}/vote").status_code == 200 + + queue = app.state.queue + assert queue.jobs.last("enrich", states=ANY_STATE) is None + assert queue.jobs.last("score", states=ANY_STATE) is None diff --git a/tests/test_readers.py b/tests/test_readers.py index 8e6d175..7791581 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -1,6 +1,7 @@ """The host modules of `plugins/readers/` and what `plugins/ats.py` does for every host: the registry and its contract check, the fetch a host is handed, -the per-role detail reads, and Lever's EU instance. No network — +the per-role detail reads the `enrich` stage makes, and Lever's EU +instance. No network — a fake host stands in where a behaviour belongs to no real host yet.""" from __future__ import annotations @@ -11,7 +12,7 @@ import subprocess import sys import types from collections.abc import Callable -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -308,34 +309,26 @@ def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: # ---------------------------------------------------------------- detail -def test_detail_is_read_for_unread_roles_only( +def test_a_pull_reads_no_role_page( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + """The pages are `enrich`'s: the pull reads the listing only, and sends + every row of a host with `detail` without a description.""" base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full", - f"{base}/2": b"two in full", f"{base}/3": b"three in full"}) - module = _fake_host(detail=_detail) - state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])} - items, state = _run(monkeypatch, module, state) - - assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"] - assert [(i.source_id, i.body) for i in items] == [ - ("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")] - # What detail returns replaces the listing's field; a role not read keeps the listing's. - assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == "" - assert json.loads(state["ats:detailed:fake:acme"]) == [ - "fake:acme:1", "fake:acme:2", "fake:acme:3"] + web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"}) + items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert web["asked"] == [f"{base}/list"] + assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")] + assert "ats:detailed:fake:acme" not in state assert state["ats:complete:fake:acme"] == "1" -def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave( +def test_a_role_read_on_its_page_keeps_the_place_its_page_gave( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: """The listing states a place on every pull, and a filled value replaces a filled column, so for a role already read it is withheld.""" base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full", - f"{base}/3": RuntimeError("down")}) - module = _fake_host(detail=lambda get, slug, row: { - "description": get(f"{base}/{row['source_id']}").decode()}) + web["answers"].update({f"{base}/list": b'["1", "3"]'}) + module = _fake_host(detail=_detail) listing = module.parse module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"} for row in listing(body, slug)] @@ -345,70 +338,25 @@ def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave( by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items} assert by_id["fake:acme:1"] == ("", "", None) - # Read now, and its page stated neither: the listing's stand. - assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid") # Never read: the listing's stand until its page is. assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid") -def test_detail_reads_stop_at_the_cap_and_the_rest_wait( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - base = "https://jobs.fake.example/acme" - ids = [str(n) for n in range(5)] - web["answers"].update({f"{base}/list": json.dumps(ids).encode()} - | {f"{base}/{i}": f"full {i}".encode() for i in ids}) - monkeypatch.setattr(ats, "DETAIL_CAP", 2) - module = _fake_host(detail=_detail) - items, state = _run(monkeypatch, module) - assert [i.body for i in items] == ["full 0", "full 1", "", "", ""] - assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"] - - web["asked"].clear() - items, state = _run(monkeypatch, module, state) - assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"] - assert [i.body for i in items] == ["", "", "full 2", "full 3", ""] - - -def test_a_detail_that_fails_skips_the_role_and_not_the_board( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one", - f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"}) - items, state = _run(monkeypatch, _fake_host(detail=_detail)) - assert [(i.source_id, i.body) for i in items] == [ - ("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")] - assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"] - assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r" - assert state["ats:complete:fake:acme"] == "1" - - -def test_a_429_on_a_detail_stops_the_reads_for_the_pull( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one", - f"{base}/2": _status(429), f"{base}/3": b"three"}) - items, state = _run(monkeypatch, _fake_host(detail=_detail)) - assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"] - assert [i.body for i in items] == ["one", "", ""] - assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] - assert state["ats:complete:fake:acme"] == "1" - - -def test_ids_no_longer_listed_leave_the_detailed_list( +def test_ids_no_longer_listed_leave_the_detail_lists( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["2"]'}) - state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]), + "ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"] + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] def test_an_incomplete_listing_keeps_the_ids_it_did_not_list( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: """A listing cut short says nothing about the roles it left out, so their - ids stay, and the next pull does not read them in detail again.""" - base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["2"]'}) + ids stay, and they are not read on their pages again.""" monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing( readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"])) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} @@ -417,6 +365,19 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list( assert state["ats:complete:fake:acme"] == "" +def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["2"]'}) + monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail)) + source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) + source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])} + roles: Any = source.fetch() # a generator, which `close` stops + assert next(roles).body == "" + roles.close() + assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"] + + def test_fetch_board_reads_the_listing_only( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" @@ -435,63 +396,137 @@ def test_a_host_without_detail_keeps_the_listings_description( assert "ats:detailed:fake:acme" not in state -def test_a_detail_answering_404_or_410_counts_as_read( +# -------------------------------------------------- read_board_details + +NOW = datetime(2026, 9, 30, 12, tzinfo=UTC) + + +def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType, + state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]: + """`ats.read_board_details` over the fake host's roles `ids`, stored + without text, in that order.""" + monkeypatch.setitem(readers.HOSTS, "fake", module) + rows = [row | {"description": ""} + for row in module.parse(json.dumps(ids).encode(), "acme")] + return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type] + now=kwargs.pop("now", NOW), **kwargs) + + +def test_details_are_read_in_the_order_given_for_unread_roles_only( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410), - f"{base}/3": b"three"}) # 1 answers 404 - items, state = _run(monkeypatch, _fake_host(detail=_detail)) - assert [i.body for i in items] == ["", "", "three"] + web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full", + f"{base}/3": b"three in full"}) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]), + "ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})} + filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"]) + + assert web["asked"] == [f"{base}/3", f"{base}/1"] + assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [ + ("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")] assert json.loads(state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:2", "fake:acme:3"] - assert json.loads(state["ats:detailfailed:fake:acme"]) == [] + + +def test_details_stop_at_the_cap_and_the_rest_wait( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + ids = [str(n) for n in range(5)] + web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids}) + module = _fake_host(detail=_detail) + state: dict[str, str] = {} + filled = _details(monkeypatch, module, state, ids, cap=2) + assert [r["description"] for r in filled] == ["full 0", "full 1"] + + web["asked"].clear() + filled = _details(monkeypatch, module, state, ids[2:], cap=2) + assert web["asked"] == [f"{base}/2", f"{base}/3"] + + +def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"), + f"{base}/3": _status(429), f"{base}/4": b"four"}) + state: dict[str, str] = {} + filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"]) + assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"] + assert [r["source_id"] for r in filled] == ["1"] + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] + # A 429 is not the role's fault. + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] @pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")]) def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: - """Two roles that always fail and a cap of two: the third is read on the - second pull, since roles never tried go before those that failed.""" + """Roles never tried go before those that failed, within the cap.""" base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure, - f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"}) - monkeypatch.setattr(ats, "DETAIL_CAP", 2) + web["answers"].update({f"{base}/1": failure, f"{base}/2": failure, + f"{base}/3": b"three", f"{base}/4": b"four"}) module = _fake_host(detail=_detail) - _items, state = _run(monkeypatch, module) + state: dict[str, str] = {} + _details(monkeypatch, module, state, ["1", "2"]) assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] web["asked"].clear() - items, state = _run(monkeypatch, module, state) - assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"] - assert [i.body for i in items] == ["", "", "three", "four"] + _details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2) + assert web["asked"] == [f"{base}/3", f"{base}/4"] - # With nothing new left, the failed ones are tried again, within the cap. web["asked"].clear() web["answers"][f"{base}/1"] = b"one at last" - items, state = _run(monkeypatch, module, state) - assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"] - assert [i.body for i in items] == ["one at last", "", "", ""] + filled = _details(monkeypatch, module, state, ["1", "2"], cap=2) + assert [r["description"] for r in filled] == ["one at last"] assert json.loads(state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:3", "fake:acme:4"] assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] -def test_a_429_on_a_detail_does_not_mark_the_role_as_failing( +def test_a_detail_answering_404_or_410_counts_as_read( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)}) - _items, state = _run(monkeypatch, _fake_host(detail=_detail)) + web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404 + state: dict[str, str] = {} + filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"]) + assert [r["source_id"] for r in filled] == ["3"] + assert json.loads(state["ats:detailed:fake:acme"]) == [ + "fake:acme:1", "fake:acme:2", "fake:acme:3"] assert json.loads(state["ats:detailfailed:fake:acme"]) == [] - assert json.loads(state["ats:detailed:fake:acme"]) == [] -def test_failed_ids_no_longer_listed_leave_the_failed_list( +def test_a_read_without_text_is_read_again_after_some_days( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + """Finding 18: an Oracle requisition not yet published has an empty page + today and a full one next week.""" base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")}) - state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} - _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) - assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] + web["answers"][f"{base}/1"] = b"" + module = _fake_host(detail=lambda get, slug, row: { + "description": get(f"{base}/{row['source_id']}").decode()}) + state: dict[str, str] = {} + assert _details(monkeypatch, module, state, ["1"]) == [] + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] + assert json.loads(state["ats:detailempty:fake:acme"]) == { + "fake:acme:1": NOW.isoformat(timespec="seconds")} + + web["asked"].clear() + later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1) + assert _details(monkeypatch, module, state, ["1"], now=later) == [] + assert web["asked"] == [] + + web["answers"][f"{base}/1"] = b"published now" + later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS) + [filled] = _details(monkeypatch, module, state, ["1"], now=later) + assert filled["description"] == "published now" + assert json.loads(state["ats:detailempty:fake:acme"]) == {} + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] + + +def test_a_role_read_before_the_dates_were_kept_is_read_again( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"]["https://jobs.fake.example/acme/1"] = b"one" + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])} + [filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"]) + assert filled["description"] == "one" def test_detail_fills_only_the_fields_a_later_pull_keeps( @@ -499,9 +534,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps( caplog: pytest.LogCaptureFixture) -> None: """`salary_text`, `department` and `title` would be rewritten from the listing on the next pull, so detail does not fill them.""" - base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1"]'}) - def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: return {"description": "in full", "remote": "hybrid", "currency": "EUR", "salary_min": 60000, "salary_max": 70000, @@ -510,7 +542,8 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps( "title": "Another title"} caplog.set_level("DEBUG", logger=_runner.log.name) - [item], _state = _run(monkeypatch, _fake_host(detail=detail)) + [row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"]) + item = ats.item_of(row, "fake:acme", "Acme") assert (item.body, item.title, item.published_at) == ( "in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC)) assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"], @@ -522,66 +555,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps( "salary_max", "posted"} -def _paged_host() -> types.ModuleType: - """A host `fake` that pages its listing, one id a page, and has `detail`.""" - module: Any = _fake_host(detail=_detail) - del module.feed_url, module.parse - - def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]: - index = int(cursor or 0) - ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}")) - return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None - - module.page = page - module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug) - assert not readers.problems(module, "fake") - return module - - -def test_a_paged_host_reads_every_page_then_each_role_once( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]', - f"{base}/1": b"one", f"{base}/2": b"two"}) - items, state = _run(monkeypatch, _paged_host()) - assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1", - f"{base}/1", f"{base}/2"] - assert [i.body for i in items] == ["one", "two"] - assert state["ats:complete:fake:acme"] == "1" - - web["asked"].clear() - items, state = _run(monkeypatch, _paged_host(), state) - assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"] - assert [i.body for i in items] == ["", ""] - - -def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429), - f"{base}/1": b"one"}) - state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]), - "ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])} - _items, state = _run(monkeypatch, _paged_host(), state) - assert state["ats:complete:fake:acme"] == "" - assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] - assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"] - - -def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - """The consumer stops after the first role, so the second, read but not - stored, is read again on the next pull.""" - base = "https://jobs.fake.example/acme" - web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"}) - monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail)) - source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) - roles: Any = source.fetch() # a generator, which `close` stops - assert next(roles).body == "one" - roles.close() - assert "ats:detailed:fake:acme" not in source.state - - # ----------------------------------------------------------------- Lever EU def test_a_lever_slug_may_name_the_eu_instance() -> None: diff --git a/tests/test_score.py b/tests/test_score.py index baed3ad..439c25e 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -1053,7 +1053,9 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise( assert batch is not None jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id) stages = [j.result.get("task") for j in jobs if j.kind == FINAL] - assert stages == list(cascade.CASCADES["pull"]) + # Enrich found no board with role pages, so the embed and score behind it + # are passed over. + assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"] work = [j.kind for j in jobs if j.kind != FINAL] # The `pull` job and the chain it asked for are one cascade, so the Jobs # tab shows a scheduled pull as one run. Extract and summarise are off and @@ -1107,10 +1109,10 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None: def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None: """By position, not by name: `score` is in the chain twice, once before extract and once after, and the second must lead on to summarise.""" - assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score", - "summarise") - assert [cascade.follows("pull", at) for at in range(6)] == \ - ["embed", "score", "extract", "score", "summarise", ""] + assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score", + "extract", "score", "summarise") + assert [cascade.follows("pull", at) for at in range(9)] == \ + ["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""] assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == "" assert cascade.follows("nope", 0) == "" assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0} @@ -1124,10 +1126,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None: made.append(t) return t - out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2}, + out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5}, context=Context()) assert out == {"task": "score", "next": "extract"} - assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3} - out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4}, + assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6} + out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7}, context=Context()) - assert out["next"] == "summarise" and made[1].carries["at"] == 5 + assert out["next"] == "summarise" and made[1].carries["at"] == 8