Merge branch 'wp-g-enrich'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
38a80d5b55
10 files changed
+944
-207
No files matched your search
@@ -66,6 +66,25 @@ class SummariseConfig(ModelTaskConfig):
|
|||||||
batch_size: int = Field(10, ge=1)
|
batch_size: int = Field(10, ge=1)
|
||||||
|
|
||||||
|
|
||||||
|
class EnrichConfig(Section):
|
||||||
|
"""Reading roles on their own pages after scoring, for the hosts whose
|
||||||
|
list carries no text. See app/jobs/enrich.py. A role is read when it has
|
||||||
|
no text and an enabled agent with a prompt ranks it ungated among its
|
||||||
|
best `top` and at `min_score` or above: the extract bar (`Limits`,
|
||||||
|
500 and 0.55) with the score a notch lower, since a role ranked on its
|
||||||
|
title alone is the one whose score its text moves most."""
|
||||||
|
|
||||||
|
enabled: bool = True
|
||||||
|
top: int = Field(500, ge=0)
|
||||||
|
min_score: float = Field(0.5, ge=0, le=1)
|
||||||
|
#: Reads per board and per run. Per board is the cap the pull had; the
|
||||||
|
#: boards are read in parallel on the `http` pool.
|
||||||
|
per_board: int = Field(200, ge=0)
|
||||||
|
per_run: int = Field(1000, ge=0)
|
||||||
|
#: Days before a page that gave no text is read again.
|
||||||
|
retry_days: int = Field(7, ge=1)
|
||||||
|
|
||||||
|
|
||||||
class Calibration(Section):
|
class Calibration(Section):
|
||||||
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
|
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
|
||||||
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
|
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
|
||||||
@@ -181,6 +200,7 @@ class Config(Section):
|
|||||||
extract: ExtractConfig = Field(default_factory=ExtractConfig)
|
extract: ExtractConfig = Field(default_factory=ExtractConfig)
|
||||||
summarise: SummariseConfig = Field(default_factory=SummariseConfig)
|
summarise: SummariseConfig = Field(default_factory=SummariseConfig)
|
||||||
embed: EmbedConfig = Field(default_factory=EmbedConfig)
|
embed: EmbedConfig = Field(default_factory=EmbedConfig)
|
||||||
|
enrich: EnrichConfig = Field(default_factory=EnrichConfig)
|
||||||
#: Starter agents, imported once into an empty table.
|
#: Starter agents, imported once into an empty table.
|
||||||
agents: list[AgentSeed] = Field(default_factory=list)
|
agents: list[AgentSeed] = Field(default_factory=list)
|
||||||
#: Starter searches, imported once into an empty set of `search`
|
#: Starter searches, imported once into an empty set of `search`
|
||||||
|
|||||||
+44
-5
@@ -1,6 +1,6 @@
|
|||||||
"""The tasks roledesk offers, and the order a pull runs them in.
|
"""The tasks roledesk offers, and the order a pull runs them in.
|
||||||
|
|
||||||
pull → embed → score → extract → score → summarise
|
pull → embed → score → enrich → embed → score → extract → score → summarise
|
||||||
|
|
||||||
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
||||||
final job of one task starts the next (`step` is the callback it carries).
|
final job of one task starts the next (`step` is the callback it carries).
|
||||||
@@ -11,7 +11,10 @@ Every stage is always in the chain, and a stage that is switched off finds
|
|||||||
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
|
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
|
||||||
satisfy, is an extract stage with no batches whose final job moves straight
|
satisfy, is an extract stage with no batches whose final job moves straight
|
||||||
on. The free stages go first: embed and score cost no tokens, and the score
|
on. The free stages go first: embed and score cost no tokens, and the score
|
||||||
they produce is what bounds the two that do. Extract reads only the best
|
they produce is what bounds the rest. Enrich reads the best roles without
|
||||||
|
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
|
||||||
|
the roles it filled; when it filled none, `step` passes over both. Extract
|
||||||
|
reads only the best
|
||||||
roles of each enabled agent, as deep and as low as that agent's own `limits`
|
roles of each enabled agent, as deep and as low as that agent's own `limits`
|
||||||
say; the second score then applies the gates to the columns it filled.
|
say; the second score then applies the gates to the columns it filled.
|
||||||
Summarise runs last, on the best roles by the same agent's summarise limits.
|
Summarise runs last, on the best roles by the same agent's summarise limits.
|
||||||
@@ -23,6 +26,11 @@ having either way.
|
|||||||
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
|
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
|
||||||
and a cascade is started by one: the `pull` job's only work is to ask for the
|
and a cascade is started by one: the `pull` job's only work is to ask for the
|
||||||
chain. Its default schedule, `every 6h`, is declared here beside it.
|
chain. Its default schedule, `every 6h`, is declared here beside it.
|
||||||
|
|
||||||
|
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
|
||||||
|
behind the `score` that follows a change to the ranking other than a pull's
|
||||||
|
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
|
||||||
|
one agent's taste a little and queues neither.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -32,16 +40,21 @@ from dataclasses import replace
|
|||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||||
from desk.tools import agent
|
from desk.tools import agent
|
||||||
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
||||||
|
|
||||||
STEP = "app.jobs.cascade:step"
|
STEP = "app.jobs.cascade:step"
|
||||||
|
|
||||||
CASCADES: dict[str, tuple[str, ...]] = {
|
CASCADES: dict[str, tuple[str, ...]] = {
|
||||||
"pull": ("pull", "embed", "score", "extract", "score", "summarise"),
|
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
|
||||||
|
"summarise"),
|
||||||
|
"enrich": ("enrich", "embed", "score"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#: What follows `enrich` for the roles it filled, passed over when it filled none.
|
||||||
|
REFILL = ("embed", "score")
|
||||||
|
|
||||||
|
|
||||||
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
||||||
"""One batch with nothing in its payload: a task that is a single job."""
|
"""One batch with nothing in its payload: a task that is a single job."""
|
||||||
@@ -91,6 +104,7 @@ STAGES: dict[str, Callable[[str], Task]] = {
|
|||||||
"score": lambda cascade: Task(kind="score", finder=_once,
|
"score": lambda cascade: Task(kind="score", finder=_once,
|
||||||
label="Scoring roles for every agent"),
|
label="Scoring roles for every agent"),
|
||||||
"summarise": _summarise,
|
"summarise": _summarise,
|
||||||
|
"enrich": lambda cascade: enrich.enrich_task(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -115,6 +129,10 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
|
|||||||
chain goes on, as in newsdesk's cascade."""
|
chain goes on, as in newsdesk's cascade."""
|
||||||
name, done = payload.get("cascade", ""), payload.get("task", "")
|
name, done = payload.get("cascade", ""), payload.get("task", "")
|
||||||
at = int(payload.get("at", -1))
|
at = int(payload.get("at", -1))
|
||||||
|
tasks = CASCADES.get(name, ())
|
||||||
|
if done == "enrich" and not enrich.filled(tally) \
|
||||||
|
and tasks[at + 1:at + 1 + len(REFILL)] == REFILL:
|
||||||
|
at += len(REFILL)
|
||||||
following = follows(name, at)
|
following = follows(name, at)
|
||||||
if following:
|
if following:
|
||||||
context.task(task(following, cascade=name, at=at + 1))
|
context.task(task(following, cascade=name, at=at + 1))
|
||||||
@@ -169,6 +187,26 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
|||||||
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
||||||
|
|
||||||
|
|
||||||
|
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
|
||||||
|
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||||
|
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
|
||||||
|
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
|
||||||
|
if not context.task(chain("enrich")):
|
||||||
|
raise RuntimeError("no queue to start the enrich cascade on")
|
||||||
|
return {"started": " > ".join(CASCADES["enrich"])}
|
||||||
|
|
||||||
|
|
||||||
|
def enrich_after(queue: Any, score_job: Any) -> Any:
|
||||||
|
"""Queue the `enrich` job behind `score_job`, the rescore that follows a
|
||||||
|
change to the ranking, however that score ends. One per score job
|
||||||
|
(`enrich:after:<id>`), so ten changes answered by one score are one
|
||||||
|
enrich. None without a score job, or when the queue refuses."""
|
||||||
|
if score_job is None:
|
||||||
|
return None
|
||||||
|
return queue.enqueue("enrich", depends_on=[score_job.id], after="done",
|
||||||
|
dedup_key=f"enrich:after:{score_job.id}")
|
||||||
|
|
||||||
|
|
||||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||||
#: is both would be listed twice by `/api/jobs/kinds`.
|
#: is both would be listed twice by `/api/jobs/kinds`.
|
||||||
@@ -182,4 +220,5 @@ for _name in ("extract", "embed", "score", "summarise"):
|
|||||||
# setting: a cadence edited or switched off under Runs survives
|
# setting: a cadence edited or switched off under Runs survives
|
||||||
# every restart (see pipeline/schedule.py).
|
# every restart (see pipeline/schedule.py).
|
||||||
declare("pull", "pull", "every 6h",
|
declare("pull", "pull", "every 6h",
|
||||||
label="Pull every board, then embed, score, extract, rescore and summarise")
|
label="Pull every board, then embed, score, read the best roles' pages, "
|
||||||
|
"extract, rescore and summarise")
|
||||||
@@ -0,0 +1,168 @@
|
|||||||
|
"""`enrich`: the best roles without text, read on their own pages.
|
||||||
|
|
||||||
|
Some hosts list a role without its text and state it on the role's own page
|
||||||
|
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
|
||||||
|
with a `detail`). The pull reads listings only; this stage reads those pages
|
||||||
|
after scoring, so the calls go where the ranking is:
|
||||||
|
|
||||||
|
1. `enrich_waiting` hands over the open roles without text that an enabled
|
||||||
|
agent with a prompt ranks ungated among its best `enrich.top` and at
|
||||||
|
`enrich.min_score` or above, best first by the highest such score. A role
|
||||||
|
below that bar for every agent gets no call.
|
||||||
|
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
|
||||||
|
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
|
||||||
|
out the roles already read, unless their read failed or gave no text
|
||||||
|
`enrich.retry_days` ago (`plugins.ats.detail_due`).
|
||||||
|
3. One batch per board, on the `http` pool, reads them
|
||||||
|
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
|
||||||
|
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
|
||||||
|
what each page filled. `Store.upsert_role` drops the vector of a role
|
||||||
|
whose text improved, so the next `embed` encodes those roles only.
|
||||||
|
|
||||||
|
In the pull cascade `embed` and `score` follow; `cascade.step` passes over
|
||||||
|
both when no role was filled. The same stage follows a rescore after an agent
|
||||||
|
is saved or a company's affinity changes (`cascade.enrich_after`), never after
|
||||||
|
a vote. Whether it runs is `stages.stage_on(config, store, "enrich")`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from collections import defaultdict
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from app.config import Config
|
||||||
|
from app.jobs import stages
|
||||||
|
from app.storage import Store
|
||||||
|
from pipeline import NULL, Batch, RunContext, Tally, Task, job
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
KIND = "enrich_batch"
|
||||||
|
SETTLE = "app.jobs.enrich:settle"
|
||||||
|
|
||||||
|
|
||||||
|
def _ats() -> Any:
|
||||||
|
"""`plugins.ats`, imported when needed, the way `pull.run_batch` does."""
|
||||||
|
from plugins import ats
|
||||||
|
|
||||||
|
return ats
|
||||||
|
|
||||||
|
|
||||||
|
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
|
||||||
|
"""A stored role back as the row its reader listed: the stored `raw`
|
||||||
|
with the host's own id, the title, the link and the date."""
|
||||||
|
try:
|
||||||
|
raw = json.loads(found.get("raw") or "{}")
|
||||||
|
except ValueError:
|
||||||
|
raw = {}
|
||||||
|
raw = raw if isinstance(raw, dict) else {}
|
||||||
|
try:
|
||||||
|
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
|
||||||
|
except ValueError:
|
||||||
|
posted = None
|
||||||
|
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
|
||||||
|
"title": found["title"], "url": found["url"], "description": "",
|
||||||
|
"posted": posted}
|
||||||
|
|
||||||
|
|
||||||
|
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
|
||||||
|
"""`(target, uids)` per board, best board first, within the budgets: the
|
||||||
|
roles `enrich_waiting` ranks, on fetchable boards whose reader has
|
||||||
|
`detail` and that are not failing, and due for a read."""
|
||||||
|
settings = config.enrich
|
||||||
|
ats = _ats()
|
||||||
|
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
|
||||||
|
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||||
|
for found in store.enrich_waiting(settings.top, settings.min_score):
|
||||||
|
target = boards.get(found["subscription_id"])
|
||||||
|
if target is None:
|
||||||
|
continue
|
||||||
|
module = ats.readers.HOSTS.get(target.partition(":")[0])
|
||||||
|
if module is None or not hasattr(module, "detail") \
|
||||||
|
or f"ats:failing:{target}" in store.state:
|
||||||
|
continue
|
||||||
|
by_board[target].append(found)
|
||||||
|
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
|
||||||
|
retry_days=settings.retry_days))
|
||||||
|
for target, found in by_board.items()}
|
||||||
|
# Best first; roles whose read failed after every other, so a page that
|
||||||
|
# keeps failing cannot take the budget of the roles never tried.
|
||||||
|
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
|
||||||
|
for target in by_board}
|
||||||
|
ranked = sorted((f for found in by_board.values() for f in found),
|
||||||
|
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
|
||||||
|
-float(f["score"] or 0), f["uid"]))
|
||||||
|
picked: dict[str, list[str]] = {}
|
||||||
|
total = 0
|
||||||
|
for found in ranked:
|
||||||
|
target = boards[found["subscription_id"]]
|
||||||
|
if found["source_id"] not in due[target]:
|
||||||
|
continue
|
||||||
|
if total >= settings.per_run:
|
||||||
|
break
|
||||||
|
uids = picked.setdefault(target, [])
|
||||||
|
if len(uids) >= settings.per_board:
|
||||||
|
continue
|
||||||
|
uids.append(found["uid"])
|
||||||
|
total += 1
|
||||||
|
return list(picked.items())
|
||||||
|
|
||||||
|
|
||||||
|
def batches(config: Config, store: Store) -> list[Batch]:
|
||||||
|
"""The finder: one batch per board, nothing while the stage is off."""
|
||||||
|
if not stages.stage_on(config, store, "enrich"):
|
||||||
|
return []
|
||||||
|
return [Batch({"target": target, "uids": uids},
|
||||||
|
label=f"Reading {len(uids)} roles on {target}")
|
||||||
|
for target, uids in plan(config, store)]
|
||||||
|
|
||||||
|
|
||||||
|
@job(KIND, "Reading roles on their own pages", resource="http")
|
||||||
|
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
|
||||||
|
context: RunContext = NULL) -> dict[str, Any]:
|
||||||
|
"""Read one board's roles in `uids` on their own pages, in that order,
|
||||||
|
and store what each page filled. `filled` is how many roles changed,
|
||||||
|
`texts` how many of them now have text."""
|
||||||
|
ats = _ats()
|
||||||
|
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
|
||||||
|
if not subs or not uids:
|
||||||
|
return {"target": target, "filled": 0, "texts": 0}
|
||||||
|
sub = subs[0]
|
||||||
|
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
|
||||||
|
rows = [_row(found[uid], target) for uid in uids if uid in found]
|
||||||
|
context.check()
|
||||||
|
with httpx.Client() as client:
|
||||||
|
filled = ats.read_board_details(store.state, target, rows, client=client,
|
||||||
|
cap=len(rows), retry_days=config.enrich.retry_days)
|
||||||
|
label = sub.company or sub.label
|
||||||
|
texts = 0
|
||||||
|
for row in filled:
|
||||||
|
item = ats.item_of(row, target, label)
|
||||||
|
item.source = "ats"
|
||||||
|
store.upsert_role(item, sub)
|
||||||
|
texts += bool(row.get("description"))
|
||||||
|
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
|
||||||
|
texts)
|
||||||
|
return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts}
|
||||||
|
|
||||||
|
|
||||||
|
def filled(tally: Tally) -> int:
|
||||||
|
"""How many roles the batches behind `tally` filled."""
|
||||||
|
return sum(int(result.get("filled") or 0) for result in tally.results)
|
||||||
|
|
||||||
|
|
||||||
|
def settle(config: Config, store: Store, tally: Tally, *,
|
||||||
|
context: RunContext = NULL) -> dict[str, Any]:
|
||||||
|
"""The run's sum: roles filled, and how many of them now have text."""
|
||||||
|
return {"filled": filled(tally),
|
||||||
|
"texts": sum(int(result.get("texts") or 0) for result in tally.results)}
|
||||||
|
|
||||||
|
|
||||||
|
def enrich_task() -> Task:
|
||||||
|
return Task(kind=KIND, finder=batches, name="enrich", settle=SETTLE,
|
||||||
|
label="Reading the best roles on their own pages")
|
||||||
+16
-7
@@ -1,4 +1,5 @@
|
|||||||
"""Whether the two model stages of a pull, `extract` and `summarise`, run.
|
"""Whether the two model stages of a pull, `extract` and `summarise`, and
|
||||||
|
the `enrich` stage run.
|
||||||
|
|
||||||
`extract.enabled` and `summarise.enabled` in config.yaml are read at startup
|
`extract.enabled` and `summarise.enabled` in config.yaml are read at startup
|
||||||
and say nothing on the page, so the switch lives in the database as well: the
|
and say nothing on the page, so the switch lives in the database as well: the
|
||||||
@@ -8,9 +9,13 @@ default for a fresh database and a deployment that never uses the switch
|
|||||||
behaves as it did. `stage_on` is the one place that decides, and both tasks'
|
behaves as it did. `stage_on` is the one place that decides, and both tasks'
|
||||||
finders ask it (`extract.extract_task`, `summarise.summarise_task`).
|
finders ask it (`extract.extract_task`, `summarise.summarise_task`).
|
||||||
|
|
||||||
`describe` is what `GET /api/stages` and `app stages` show: for each stage
|
`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`)
|
||||||
whether it is on, where that comes from, the model it uses, and who would
|
asks no model and is on by default (`enrich.enabled`); its switch is
|
||||||
answer.
|
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
|
||||||
|
|
||||||
|
`describe` is what `GET /api/stages` and `app stages` show: for each model
|
||||||
|
stage whether it is on, where that comes from, the model it uses, and who
|
||||||
|
would answer.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -25,6 +30,9 @@ from desk.tools import agent
|
|||||||
|
|
||||||
NAMES = ("extract", "summarise")
|
NAMES = ("extract", "summarise")
|
||||||
|
|
||||||
|
#: Every stage with a switch: the model stages and `enrich`.
|
||||||
|
SWITCHES = (*NAMES, "enrich")
|
||||||
|
|
||||||
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
|
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
|
||||||
"see docs/llm-backends.md")
|
"see docs/llm-backends.md")
|
||||||
ANSWERS = {
|
ANSWERS = {
|
||||||
@@ -39,7 +47,7 @@ class Stage(BaseModel):
|
|||||||
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
|
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
|
||||||
it in words."""
|
it in words."""
|
||||||
|
|
||||||
name: Literal["extract", "summarise"]
|
name: Literal["extract", "summarise", "enrich"]
|
||||||
on: bool
|
on: bool
|
||||||
source: Literal["stored", "config"]
|
source: Literal["stored", "config"]
|
||||||
model: str
|
model: str
|
||||||
@@ -65,7 +73,7 @@ def stage_on(config: Config, store: Store, name: str) -> bool:
|
|||||||
|
|
||||||
def set_stage(store: Store, name: str, on: bool) -> None:
|
def set_stage(store: Store, name: str, on: bool) -> None:
|
||||||
"""Store the switch. A `KeyError` for a name that is not a stage."""
|
"""Store the switch. A `KeyError` for a name that is not a stage."""
|
||||||
if name not in NAMES:
|
if name not in SWITCHES:
|
||||||
raise KeyError(name)
|
raise KeyError(name)
|
||||||
store.state[_key(name)] = "on" if on else "off"
|
store.state[_key(name)] = "on" if on else "off"
|
||||||
|
|
||||||
@@ -84,7 +92,8 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
|
|||||||
backend, said = answer(config)
|
backend, said = answer(config)
|
||||||
return Stage(name=name, on=stage_on(config, store, name),
|
return Stage(name=name, on=stage_on(config, store, name),
|
||||||
source="config" if stored(store, name) is None else "stored",
|
source="config" if stored(store, name) is None else "stored",
|
||||||
model=getattr(config, name).model, backend=backend, answer=said)
|
model=getattr(getattr(config, name), "model", ""), backend=backend,
|
||||||
|
answer=said)
|
||||||
|
|
||||||
|
|
||||||
def describe(config: Config, store: Store) -> list[Stage]:
|
def describe(config: Config, store: Store) -> list[Stage]:
|
||||||
|
|||||||
@@ -1503,6 +1503,46 @@ class Store:
|
|||||||
return None
|
return None
|
||||||
return self.role(uid)
|
return self.role(uid)
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- enrich
|
||||||
|
|
||||||
|
def enrich_waiting(self, top: int, floor: float,
|
||||||
|
uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
|
||||||
|
"""Open `ats` roles without text that an enabled agent with a prompt
|
||||||
|
ranks ungated among its best `top` and at `floor` or above, best
|
||||||
|
first by the highest such score, as `{uid, source_id,
|
||||||
|
subscription_id, title, url, published_at, raw, score}`. The ranking
|
||||||
|
is `_top_scope`'s, with the bar given rather than the agent's own.
|
||||||
|
Given `uids`, those of them still without text, whatever their rank,
|
||||||
|
in no order (`score` is None), since a batch finishes what it was
|
||||||
|
dealt."""
|
||||||
|
fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, "
|
||||||
|
"r.published_at, r.raw")
|
||||||
|
waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source = 'ats' "
|
||||||
|
"AND r.subscription_id IS NOT NULL AND r.description = '' "
|
||||||
|
"AND r.article_text = ''")
|
||||||
|
if uids is not None:
|
||||||
|
rows = self._rows(
|
||||||
|
f"SELECT {fields}, NULL AS score FROM roles r WHERE {waiting} "
|
||||||
|
"AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),))
|
||||||
|
else:
|
||||||
|
rows = self._rows(
|
||||||
|
f"""SELECT {fields}, MAX(ranked.score) AS score FROM (
|
||||||
|
SELECT s.uid, s.score, ROW_NUMBER() OVER (
|
||||||
|
PARTITION BY s.agent_id
|
||||||
|
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
||||||
|
FROM scores s
|
||||||
|
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||||
|
AND EXISTS (SELECT 1 FROM agent_prompts p
|
||||||
|
WHERE p.agent_id = a.id
|
||||||
|
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
|
||||||
|
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||||
|
AND top.dup_of IS NULL
|
||||||
|
WHERE s.gated_reason = '') ranked
|
||||||
|
JOIN roles r ON r.uid = ranked.uid
|
||||||
|
WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting}
|
||||||
|
GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor))
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
# -------------------------------------------------------------- vectors
|
# -------------------------------------------------------------- vectors
|
||||||
|
|
||||||
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
||||||
|
|||||||
+7
-4
@@ -681,6 +681,7 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep,
|
|||||||
except KeyError:
|
except KeyError:
|
||||||
raise HTTPException(404, f"no such company: {key}") from None
|
raise HTTPException(404, f"no such company: {key}") from None
|
||||||
job = _score_after_change(queue)
|
job = _score_after_change(queue)
|
||||||
|
jobs.cascade.enrich_after(queue, job)
|
||||||
return AffinityOut(**company.model_dump(), score_job=job.id if job else None)
|
return AffinityOut(**company.model_dump(), score_job=job.id if job else None)
|
||||||
|
|
||||||
|
|
||||||
@@ -965,17 +966,19 @@ class AgentBody(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
@router.post("/agents")
|
@router.post("/agents")
|
||||||
def save_agent(body: AgentBody, store: StoreDep) -> Agent:
|
def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent:
|
||||||
"""Scores are not recomputed here: the page asks `POST /api/score` after
|
"""Queues the rescore, which the page's `POST /api/score` after a save
|
||||||
a save and waits on the job it answers with."""
|
joins, and the enrich run behind it (`cascade.enrich_after`)."""
|
||||||
try:
|
try:
|
||||||
return store.save_agent(body.name, body.intent, body.gates, body.weights,
|
saved = store.save_agent(body.name, body.intent, body.gates, body.weights,
|
||||||
body.enabled, agent_id=body.id, limits=body.limits,
|
body.enabled, agent_id=body.id, limits=body.limits,
|
||||||
prompts=body.prompts)
|
prompts=body.prompts)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(409, str(exc)) from exc
|
raise HTTPException(409, str(exc)) from exc
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise HTTPException(404, f"no such agent: {body.id}") from None
|
raise HTTPException(404, f"no such agent: {body.id}") from None
|
||||||
|
jobs.cascade.enrich_after(queue, _score_after_change(queue))
|
||||||
|
return saved
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/agents/{agent_id}", status_code=204)
|
@router.delete("/agents/{agent_id}", status_code=204)
|
||||||
|
|||||||
+134
-19
@@ -20,6 +20,10 @@ recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
|||||||
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
|
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
|
||||||
filter, narrowed to what its reader declares (`_runner.ask_for`).
|
filter, narrowed to what its reader declares (`_runner.ask_for`).
|
||||||
|
|
||||||
|
A pull reads listings only. A host that states a role's text on the role's
|
||||||
|
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
||||||
|
scoring, best roles first, through `read_board_details`.
|
||||||
|
|
||||||
The limits below are read when a board is read, so a test that replaces one
|
The limits below are read when a board is read, so a test that replaces one
|
||||||
on this module reaches the runner.
|
on this module reaches the runner.
|
||||||
"""
|
"""
|
||||||
@@ -31,7 +35,7 @@ import logging
|
|||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
from collections.abc import Callable, Iterator
|
from collections.abc import Callable, Iterator
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -363,21 +367,141 @@ class ATSSource(Source):
|
|||||||
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
||||||
return f"{host}:{slug}:{source_id}"
|
return f"{host}:{slug}:{source_id}"
|
||||||
|
|
||||||
|
# A host with `detail` states a role's text on the role's own page,
|
||||||
|
# which the `enrich` stage reads after scoring (`read_board_details`),
|
||||||
|
# never the pull. Its rows go out without a description, so the text
|
||||||
|
# a role's page gave stays, and a role read on its own page goes out
|
||||||
|
# without the listing's place and remote as well, so what its page
|
||||||
|
# said stays too (see `_runner.read_details`).
|
||||||
rows = listing.rows
|
rows = listing.rows
|
||||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
|
||||||
detail_sets: tuple[set[str], set[str]] | None = None
|
detail_sets: tuple[set[str], set[str]] | None = None
|
||||||
if hasattr(module, "detail"):
|
if hasattr(module, "detail"):
|
||||||
detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys)
|
detailed, failed = (_runner.id_set(self.state, key)
|
||||||
listed = {compound(i) for i in listing.ids} if listing.complete else None
|
for key in detail_keys(target)[:2])
|
||||||
rows, detailed, failed = _runner.read_details(
|
rows = [row | {"description": ""}
|
||||||
module, host, slug, get, rows, detailed, failed, listed=listed,
|
| (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {})
|
||||||
limits=_limits())
|
for row in rows]
|
||||||
detail_sets = (detailed, failed)
|
if listing.complete:
|
||||||
|
listed = {compound(i) for i in listing.ids}
|
||||||
|
detail_sets = (detailed & listed, failed & listed)
|
||||||
|
|
||||||
for row in rows:
|
for row in rows:
|
||||||
|
yield item_of(row, target, label)
|
||||||
|
# Stamped only once every row went out, so a board that blew up half
|
||||||
|
# way through is neither marked ok nor has a partial id list treated
|
||||||
|
# as the whole of it.
|
||||||
|
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||||
|
[compound(i) for i in listing.ids])
|
||||||
|
if detail_sets is not None:
|
||||||
|
# A complete listing drops the ids it no longer lists.
|
||||||
|
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||||
|
if key in self.state:
|
||||||
|
self.state[key] = json.dumps(sorted(ids))
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------ the per-role reads
|
||||||
|
|
||||||
|
#: What a pull withholds for a role already read on its own page: the
|
||||||
|
#: listing's place and remote, which would replace what the page stated.
|
||||||
|
_KEPT_FROM_DETAIL: dict[str, Any] = {"location": "", "places": [], "remote": None}
|
||||||
|
|
||||||
|
#: Days before a role whose own page gave no text is read again: an Oracle
|
||||||
|
#: requisition not yet published, an empty SuccessFactors block. A 404 or a
|
||||||
|
#: 410 counts the same, since the listing closes a role that is gone.
|
||||||
|
EMPTY_RETRY_DAYS = 7
|
||||||
|
|
||||||
|
|
||||||
|
def detail_keys(target: str) -> tuple[str, str, str]:
|
||||||
|
"""The state keys of one board's per-role reads: the ids read, the ids
|
||||||
|
whose read failed, and `{id: date}` of the reads that gave no text."""
|
||||||
|
return (f"ats:detailed:{target}", f"ats:detailfailed:{target}",
|
||||||
|
f"ats:detailempty:{target}")
|
||||||
|
|
||||||
|
|
||||||
|
def _dates(state: Any, key: str) -> dict[str, str]:
|
||||||
|
try:
|
||||||
|
value = json.loads(state.get(key) or "{}")
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return {}
|
||||||
|
return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def detail_due(state: Any, target: str, ids: list[str], *, now: datetime | None = None,
|
||||||
|
retry_days: int = EMPTY_RETRY_DAYS) -> list[str]:
|
||||||
|
"""Of `ids` (compound, `host:slug:id`, roles without text), those to read
|
||||||
|
on their own page, in the order given: never read, failed, or read with
|
||||||
|
no text at least `retry_days` ago. A role read before its date was kept
|
||||||
|
counts as read that long ago."""
|
||||||
|
detailed = _runner.id_set(state, detail_keys(target)[0])
|
||||||
|
empty = _dates(state, detail_keys(target)[2])
|
||||||
|
cutoff = ((now or datetime.now(UTC)) - timedelta(days=retry_days)).isoformat(
|
||||||
|
timespec="seconds")
|
||||||
|
return [i for i in ids if i not in detailed or empty.get(i, "") <= cutoff]
|
||||||
|
|
||||||
|
|
||||||
|
def read_board_details(state: Any, target: str, rows: list[Row], *, client: httpx.Client,
|
||||||
|
cap: int | None = None, now: datetime | None = None,
|
||||||
|
retry_days: int = EMPTY_RETRY_DAYS) -> list[Row]:
|
||||||
|
"""Read `rows` of one board on their own pages, in the order given with
|
||||||
|
the roles whose read failed last, at most `cap` (`DETAIL_CAP`), and keep the board's per-role state
|
||||||
|
(`detail_keys`). The rows are the stored roles without text, each with
|
||||||
|
its host's own `source_id`. Returns the rows whose page filled a field,
|
||||||
|
each merged with what it filled.
|
||||||
|
|
||||||
|
A read that gave no text is kept with its date and read again once it is
|
||||||
|
`retry_days` old (`detail_due`). The sets are read again just before they
|
||||||
|
are written and changed by what this read did, so a pull that dropped ids
|
||||||
|
meanwhile keeps its drop."""
|
||||||
|
host, _, slug = target.partition(":")
|
||||||
|
module = readers.HOSTS[host]
|
||||||
|
keys = detail_keys(target)
|
||||||
|
now = now or datetime.now(UTC)
|
||||||
|
|
||||||
|
def compound(row: Row) -> str:
|
||||||
|
return f"{host}:{slug}:{row['source_id']}"
|
||||||
|
|
||||||
|
due = set(detail_due(state, target, [compound(r) for r in rows], now=now,
|
||||||
|
retry_days=retry_days))
|
||||||
|
detailed, failed = (_runner.id_set(state, key) for key in keys[:2])
|
||||||
|
# Roles whose read failed go after the rest, so they cannot starve them.
|
||||||
|
wanted = sorted((row for row in rows if compound(row) in due),
|
||||||
|
key=lambda row: compound(row) in failed)
|
||||||
|
wanted = wanted[:DETAIL_CAP if cap is None else cap]
|
||||||
|
if not wanted:
|
||||||
|
return []
|
||||||
|
again = {compound(row) for row in wanted} & detailed
|
||||||
|
limits = _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=len(wanted))
|
||||||
|
out, done, failing = _runner.read_details(
|
||||||
|
module, host, slug, _getter(client, host, slug), wanted, detailed - again, failed,
|
||||||
|
limits=limits)
|
||||||
|
read = done - (detailed - again)
|
||||||
|
filled: list[Row] = []
|
||||||
|
empty = _dates(state, keys[2])
|
||||||
|
for before, after in zip(wanted, out, strict=True):
|
||||||
|
key = compound(before)
|
||||||
|
if key in read:
|
||||||
|
if after.get("description"):
|
||||||
|
empty.pop(key, None)
|
||||||
|
else:
|
||||||
|
empty[key] = now.isoformat(timespec="seconds")
|
||||||
|
if after != before | {"description": ""}:
|
||||||
|
filled.append(after)
|
||||||
|
|
||||||
|
detailed_now = (_runner.id_set(state, keys[0]) - again) | read
|
||||||
|
failed_now = (_runner.id_set(state, keys[1]) | (failing - failed)) - read
|
||||||
|
state[keys[0]] = json.dumps(sorted(detailed_now))
|
||||||
|
state[keys[1]] = json.dumps(sorted(failed_now))
|
||||||
|
state[keys[2]] = json.dumps({k: v for k, v in sorted(empty.items()) if k in detailed_now})
|
||||||
|
return filled
|
||||||
|
|
||||||
|
|
||||||
|
def item_of(row: Row, target: str, label: str) -> Item:
|
||||||
|
"""One row of board `target` as the `Item` the store takes: its id
|
||||||
|
compound, `host:slug:id`, as `roles.source_id` has it."""
|
||||||
|
host, _, slug = target.partition(":")
|
||||||
description = row.get("description") or ""
|
description = row.get("description") or ""
|
||||||
yield Item(
|
return Item(
|
||||||
source_id=compound(row["source_id"]),
|
source_id=f"{host}:{slug}:{row['source_id']}",
|
||||||
location=row["url"],
|
location=row["url"],
|
||||||
title=row["title"],
|
title=row["title"],
|
||||||
excerpt=description[:300],
|
excerpt=description[:300],
|
||||||
@@ -392,12 +516,3 @@ class ATSSource(Source):
|
|||||||
"slug": slug,
|
"slug": slug,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
# Stamped only once every row went out, so a board that blew up half
|
|
||||||
# way through is neither marked ok nor has a partial id list treated
|
|
||||||
# as the whole of it, and a pull stopped half way reads the rest of
|
|
||||||
# its details again.
|
|
||||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
|
||||||
[compound(i) for i in listing.ids])
|
|
||||||
if detail_sets is not None:
|
|
||||||
for key, ids in zip(detail_keys, detail_sets, strict=True):
|
|
||||||
self.state[key] = json.dumps(sorted(ids))
|
|
||||||
@@ -0,0 +1,368 @@
|
|||||||
|
"""The `enrich` stage: the best roles without text, read on their own pages
|
||||||
|
after scoring. Which roles and in what order, the budgets, what is stored,
|
||||||
|
the cascade around it, and what triggers it. No network: a fake host stands
|
||||||
|
in, answering from a dict."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import types
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from app.config import Config, ServerConfig # noqa: E402
|
||||||
|
from app.jobs import cascade, embed, enrich, stages # noqa: E402
|
||||||
|
from app.main import create_app # noqa: E402
|
||||||
|
from app.storage import Store # noqa: E402
|
||||||
|
from desk.models import Item # noqa: E402
|
||||||
|
from pipeline import FINAL, JobQueue, Jobs, Tally # noqa: E402
|
||||||
|
from plugins import ats, readers # noqa: E402
|
||||||
|
|
||||||
|
BASE = "https://jobs.fake.example"
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_host(name: str = "fake", *, detail: bool = True) -> types.ModuleType:
|
||||||
|
"""A host whose role pages are `BASE/<slug>/<id>`, answered by `web`."""
|
||||||
|
module: Any = types.ModuleType(name)
|
||||||
|
module.NAME, module.LABEL, module.PATTERNS = name, name.title(), ()
|
||||||
|
module.slug_ok = lambda slug: slug.isalnum()
|
||||||
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||||
|
module.careers_url = lambda slug: f"{BASE}/{slug}"
|
||||||
|
module.feed_url = lambda slug: f"{BASE}/{slug}/list"
|
||||||
|
module.parse = lambda body, slug: []
|
||||||
|
if detail:
|
||||||
|
module.detail = lambda get, slug, row: {
|
||||||
|
"description": get(f"{BASE}/{slug}/{row['source_id']}").decode(),
|
||||||
|
"location": "Copenhagen"}
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||||
|
"""`ats._get` answering from `answers` (URL -> body), 404 otherwise,
|
||||||
|
remembering what it was asked; hosts `fake` and `plain` (no detail)."""
|
||||||
|
answers: dict[str, Any] = {}
|
||||||
|
asked: list[str] = []
|
||||||
|
|
||||||
|
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
|
||||||
|
asked.append(url)
|
||||||
|
answer = answers.get(url)
|
||||||
|
if answer is None:
|
||||||
|
request = httpx.Request("GET", url)
|
||||||
|
raise httpx.HTTPStatusError("404", request=request,
|
||||||
|
response=httpx.Response(404, request=request))
|
||||||
|
return answer
|
||||||
|
|
||||||
|
monkeypatch.setattr(ats, "_get", fake_get)
|
||||||
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host())
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "plain", _fake_host("plain", detail=False))
|
||||||
|
return {"answers": answers, "asked": asked}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def store(tmp_path: Path) -> Store:
|
||||||
|
return Store(tmp_path / "roledesk.db")
|
||||||
|
|
||||||
|
|
||||||
|
def config(tmp_path: Path, **enrich_settings: Any) -> Config:
|
||||||
|
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"},
|
||||||
|
enrich=enrich_settings)
|
||||||
|
|
||||||
|
|
||||||
|
def board(store: Store, target: str) -> Any:
|
||||||
|
host = target.partition(":")[0]
|
||||||
|
return store.add_subscription("ats", target, label=target, company=target, board=host)
|
||||||
|
|
||||||
|
|
||||||
|
def role(store: Store, target: str, native: str, *, body: str = "",
|
||||||
|
title: str = "Backend Engineer") -> str:
|
||||||
|
"""A role of board `target` as a pull stores it: without text on a host
|
||||||
|
with `detail`."""
|
||||||
|
sub = next(s for s in store.subscriptions(source="ats") if s.target == target)
|
||||||
|
host, _, slug = target.partition(":")
|
||||||
|
item = Item(source_id=f"{target}:{native}", location=f"{BASE}/{slug}/{native}",
|
||||||
|
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||||
|
raw={"company": target, "place": "2 Locations", "board": host, "slug": slug})
|
||||||
|
item.source = "ats"
|
||||||
|
store.upsert_role(item, sub)
|
||||||
|
return item.uid
|
||||||
|
|
||||||
|
|
||||||
|
def scored(store: Store, name: str, scores: dict[str, float], *,
|
||||||
|
gated: tuple[str, ...] = ()) -> Any:
|
||||||
|
agent = store.save_agent(name, f"{name} python backend")
|
||||||
|
store.replace_scores(agent.id, [(uid, value, "gated" if uid in gated else "")
|
||||||
|
for uid, value in scores.items()])
|
||||||
|
return agent
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------------- which roles
|
||||||
|
|
||||||
|
def test_roles_go_best_first_by_their_best_agent_score(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
board(store, "fake:acme")
|
||||||
|
board(store, "fake:beta")
|
||||||
|
a1, a2 = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
|
||||||
|
b1 = role(store, "fake:beta", "1")
|
||||||
|
scored(store, "One", {a1: 0.6, a2: 0.9, b1: 0.55})
|
||||||
|
scored(store, "Two", {a1: 0.95, b1: 0.7})
|
||||||
|
|
||||||
|
assert [(r["uid"], r["score"]) for r in store.enrich_waiting(500, 0.5)] == [
|
||||||
|
(a1, 0.95), (a2, 0.9), (b1, 0.7)]
|
||||||
|
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]),
|
||||||
|
("fake:beta", [b1])]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_no_agent_ranks_high_enough_gets_no_call(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
board(store, "fake:acme")
|
||||||
|
low, gated, deep, kept = (role(store, "fake:acme", n) for n in ("1", "2", "3", "4"))
|
||||||
|
scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,))
|
||||||
|
# `top` 1 per agent: only the best ungated role of each agent is in reach.
|
||||||
|
assert enrich.plan(config(tmp_path, top=1), store) == [("fake:acme", [kept])]
|
||||||
|
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])]
|
||||||
|
# A disabled agent ranks nothing.
|
||||||
|
agent = store.agents()[0]
|
||||||
|
store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, enabled=False,
|
||||||
|
agent_id=agent.id, limits=agent.limits, prompts=agent.prompts)
|
||||||
|
assert enrich.plan(config(tmp_path), store) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
for target in ("fake:acme", "plain:acme", "fake:down"):
|
||||||
|
board(store, target)
|
||||||
|
wanted = role(store, "fake:acme", "1")
|
||||||
|
has_text = role(store, "fake:acme", "2", body="Python in full")
|
||||||
|
plain = role(store, "plain:acme", "1")
|
||||||
|
down = role(store, "fake:down", "1")
|
||||||
|
store.state["ats:failing:fake:down"] = json.dumps({"since": "2026-09-29", "reason": "500"})
|
||||||
|
scored(store, "One", {wanted: 0.9, has_text: 0.95, plain: 0.9, down: 0.9})
|
||||||
|
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [wanted])]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
board(store, "fake:acme")
|
||||||
|
read, failed, fresh = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
|
||||||
|
now = datetime.now(UTC).isoformat(timespec="seconds")
|
||||||
|
store.state["ats:detailed:fake:acme"] = json.dumps(["fake:acme:1"])
|
||||||
|
store.state["ats:detailempty:fake:acme"] = json.dumps({"fake:acme:1": now})
|
||||||
|
store.state["ats:detailfailed:fake:acme"] = json.dumps(["fake:acme:2"])
|
||||||
|
scored(store, "One", {read: 0.9, failed: 0.8, fresh: 0.6})
|
||||||
|
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])]
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ budgets
|
||||||
|
|
||||||
|
def test_the_budgets_per_board_and_per_run(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
board(store, "fake:acme")
|
||||||
|
board(store, "fake:beta")
|
||||||
|
acme = [role(store, "fake:acme", str(n)) for n in range(4)]
|
||||||
|
beta = [role(store, "fake:beta", str(n)) for n in range(4)]
|
||||||
|
# Interleaved: acme 0.99, beta 0.98, acme 0.97, ...
|
||||||
|
scored(store, "One", {uid: 0.99 - 0.01 * n
|
||||||
|
for n, uid in enumerate(u for pair in zip(acme, beta, strict=True)
|
||||||
|
for u in pair)})
|
||||||
|
|
||||||
|
assert enrich.plan(config(tmp_path, per_board=2), store) == [
|
||||||
|
("fake:acme", acme[:2]), ("fake:beta", beta[:2])]
|
||||||
|
assert enrich.plan(config(tmp_path, per_run=3), store) == [
|
||||||
|
("fake:acme", acme[:2]), ("fake:beta", beta[:1])]
|
||||||
|
assert enrich.plan(config(tmp_path, per_board=1, per_run=5), store) == [
|
||||||
|
("fake:acme", acme[:1]), ("fake:beta", beta[:1])]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_stage_off_finds_nothing(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
board(store, "fake:acme")
|
||||||
|
scored(store, "One", {role(store, "fake:acme", "1"): 0.9})
|
||||||
|
assert len(enrich.batches(config(tmp_path), store)) == 1
|
||||||
|
assert enrich.batches(config(tmp_path, enabled=False), store) == []
|
||||||
|
stages.set_stage(store, "enrich", True)
|
||||||
|
assert len(enrich.batches(config(tmp_path, enabled=False), store)) == 1
|
||||||
|
stages.set_stage(store, "enrich", False)
|
||||||
|
assert enrich.batches(config(tmp_path), store) == []
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ what is stored
|
||||||
|
|
||||||
|
def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
board(store, "fake:acme")
|
||||||
|
one, two, gone = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
|
||||||
|
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32)) for uid in (one, two, gone)])
|
||||||
|
web["answers"].update({f"{BASE}/acme/1": b"Python in full", f"{BASE}/acme/2": b"Go too"})
|
||||||
|
|
||||||
|
out = enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one])
|
||||||
|
|
||||||
|
assert out == {"target": "fake:acme", "read": 3, "filled": 2, "texts": 2}
|
||||||
|
assert web["asked"] == [f"{BASE}/acme/2", f"{BASE}/acme/3", f"{BASE}/acme/1"]
|
||||||
|
first = store.role(one)
|
||||||
|
assert first is not None and first.description == "Python in full"
|
||||||
|
assert first.place == "Copenhagen" and first.title == "Backend Engineer"
|
||||||
|
assert set(store.vectors("fake")) == {gone}
|
||||||
|
assert json.loads(store.state["ats:detailed:fake:acme"]) == [
|
||||||
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||||
|
# A later pull of the same listing keeps the text and the page's place.
|
||||||
|
item = ats.item_of({"source_id": "1", "title": "Backend Engineer", "url": f"{BASE}/acme/1",
|
||||||
|
"description": "", "location": "", "places": [], "remote": None},
|
||||||
|
"fake:acme", "fake:acme")
|
||||||
|
item.source = "ats"
|
||||||
|
store.upsert_role(item, store.subscriptions(source="ats")[0])
|
||||||
|
again = store.role(one)
|
||||||
|
assert again is not None and again.description == "Python in full"
|
||||||
|
assert again.place == "Copenhagen"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_switched_off_since_the_plan_reads_nothing(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
sub = board(store, "fake:acme")
|
||||||
|
uid = role(store, "fake:acme", "1")
|
||||||
|
store.update_subscription(sub.id, fetch=0)
|
||||||
|
assert enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0
|
||||||
|
assert web["asked"] == []
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ cascade
|
||||||
|
|
||||||
|
class _Context:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.made: list[Any] = []
|
||||||
|
|
||||||
|
def task(self, t: Any) -> Any:
|
||||||
|
self.made.append(t)
|
||||||
|
return t
|
||||||
|
|
||||||
|
|
||||||
|
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
|
||||||
|
context = _Context()
|
||||||
|
empty = Tally(results=({"filled": 0},))
|
||||||
|
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
|
||||||
|
context=context)
|
||||||
|
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
|
||||||
|
|
||||||
|
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
|
||||||
|
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
|
||||||
|
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
|
||||||
|
|
||||||
|
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
|
||||||
|
context=context)
|
||||||
|
assert out["next"] == "" and len(context.made) == 2
|
||||||
|
|
||||||
|
|
||||||
|
class _Embedder:
|
||||||
|
def encode(self, texts: list[str]) -> np.ndarray:
|
||||||
|
return np.vstack([np.asarray([t.lower().count("python"), 0.1], dtype=np.float32)
|
||||||
|
for t in texts])
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any],
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(embed, "load_embedder", lambda model: _Embedder())
|
||||||
|
board(store, "fake:acme")
|
||||||
|
one, two = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
|
||||||
|
store.put_vectors("fake", [(uid, np.ones(2, dtype=np.float32)) for uid in (one, two)])
|
||||||
|
agent = scored(store, "One", {one: 0.9, two: 0.1})
|
||||||
|
web["answers"][f"{BASE}/acme/1"] = b"Python in full"
|
||||||
|
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config(tmp_path), store,
|
||||||
|
workers={"cpu": 1, "http": 1, "llm": 1})
|
||||||
|
|
||||||
|
started = queue.enqueue("enrich")
|
||||||
|
queue.drain(timeout=60)
|
||||||
|
|
||||||
|
assert started is not None
|
||||||
|
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
|
||||||
|
key=lambda j: j.id)
|
||||||
|
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
|
||||||
|
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
|
||||||
|
assert web["asked"] == [f"{BASE}/acme/1"]
|
||||||
|
embedded = next(j for j in jobs if j.kind == "embed")
|
||||||
|
assert embedded.result["embedded"] == 1
|
||||||
|
assert set(store.vectors("fake")) == {one, two}
|
||||||
|
assert store.scores(agent.id)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------- triggers
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def app(tmp_path: Path) -> FastAPI:
|
||||||
|
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
||||||
|
database=tmp_path / "roledesk.db"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client(app: FastAPI) -> Iterator[TestClient]:
|
||||||
|
with TestClient(app) as c:
|
||||||
|
yield c
|
||||||
|
|
||||||
|
|
||||||
|
ANY_STATE = ("queued", "running", "done", "failed", "cancelled")
|
||||||
|
|
||||||
|
|
||||||
|
def _enrich_behind_a_score(queue: Any) -> Any:
|
||||||
|
found = queue.jobs.last("enrich", states=ANY_STATE)
|
||||||
|
assert found is not None
|
||||||
|
assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"]
|
||||||
|
assert found.after == "done"
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def test_saving_an_agent_queues_a_score_then_enrich(client: TestClient, app: FastAPI) -> None:
|
||||||
|
res = client.post("/api/agents", json={"name": "Backend", "intent": "python backend"})
|
||||||
|
assert res.status_code == 200
|
||||||
|
_enrich_behind_a_score(app.state.queue)
|
||||||
|
res = client.post("/api/agents", json={"id": res.json()["id"], "name": "Backend",
|
||||||
|
"intent": "go backend"})
|
||||||
|
assert res.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_company_affinity_queues_a_score_then_enrich(
|
||||||
|
client: TestClient, app: FastAPI) -> None:
|
||||||
|
store: Store = app.state.store
|
||||||
|
store.save_agent("Backend", "python backend")
|
||||||
|
store.add_company("Acme")
|
||||||
|
res = client.post("/api/companies/acme/affinity", json={"affinity": 2})
|
||||||
|
assert res.status_code == 200
|
||||||
|
found = _enrich_behind_a_score(app.state.queue)
|
||||||
|
assert found.depends_on == [res.json()["score_job"]]
|
||||||
|
|
||||||
|
|
||||||
|
def test_one_score_is_one_enrich(app: FastAPI) -> None:
|
||||||
|
queue = app.state.queue
|
||||||
|
score = queue.enqueue("score")
|
||||||
|
first = cascade.enrich_after(queue, score)
|
||||||
|
assert cascade.enrich_after(queue, score).id == first.id
|
||||||
|
assert cascade.enrich_after(queue, None) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_vote_queues_neither_score_nor_enrich(client: TestClient, app: FastAPI) -> None:
|
||||||
|
store: Store = app.state.store
|
||||||
|
store.save_agent("Backend", "python backend")
|
||||||
|
item = Item(source_id="fake:acme:1", location=f"{BASE}/acme/1", title="Backend Engineer",
|
||||||
|
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme"})
|
||||||
|
item.source = "ats"
|
||||||
|
store.upsert_role(item)
|
||||||
|
|
||||||
|
assert client.post(f"/api/roles/{item.uid}/vote", json={"value": 1}).status_code == 200
|
||||||
|
assert client.delete(f"/api/roles/{item.uid}/vote").status_code == 200
|
||||||
|
|
||||||
|
queue = app.state.queue
|
||||||
|
assert queue.jobs.last("enrich", states=ANY_STATE) is None
|
||||||
|
assert queue.jobs.last("score", states=ANY_STATE) is None
|
||||||
+136
-163
@@ -1,6 +1,7 @@
|
|||||||
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
||||||
every host: the registry and its contract check, the fetch a host is handed,
|
every host: the registry and its contract check, the fetch a host is handed,
|
||||||
the per-role detail reads, and Lever's EU instance. No network —
|
the per-role detail reads the `enrich` stage makes, and Lever's EU
|
||||||
|
instance. No network —
|
||||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -11,7 +12,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
import types
|
import types
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -308,34 +309,26 @@ def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
|||||||
|
|
||||||
# ---------------------------------------------------------------- detail
|
# ---------------------------------------------------------------- detail
|
||||||
|
|
||||||
def test_detail_is_read_for_unread_roles_only(
|
def test_a_pull_reads_no_role_page(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
"""The pages are `enrich`'s: the pull reads the listing only, and sends
|
||||||
|
every row of a host with `detail` without a description."""
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
|
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"})
|
||||||
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
|
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||||
module = _fake_host(detail=_detail)
|
assert web["asked"] == [f"{base}/list"]
|
||||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
|
assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")]
|
||||||
items, state = _run(monkeypatch, module, state)
|
assert "ats:detailed:fake:acme" not in state
|
||||||
|
|
||||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
|
|
||||||
assert [(i.source_id, i.body) for i in items] == [
|
|
||||||
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
|
|
||||||
# What detail returns replaces the listing's field; a role not read keeps the listing's.
|
|
||||||
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
|
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
|
||||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
|
||||||
assert state["ats:complete:fake:acme"] == "1"
|
assert state["ats:complete:fake:acme"] == "1"
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
|
def test_a_role_read_on_its_page_keeps_the_place_its_page_gave(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
"""The listing states a place on every pull, and a filled value replaces
|
"""The listing states a place on every pull, and a filled value replaces
|
||||||
a filled column, so for a role already read it is withheld."""
|
a filled column, so for a role already read it is withheld."""
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full",
|
web["answers"].update({f"{base}/list": b'["1", "3"]'})
|
||||||
f"{base}/3": RuntimeError("down")})
|
module = _fake_host(detail=_detail)
|
||||||
module = _fake_host(detail=lambda get, slug, row: {
|
|
||||||
"description": get(f"{base}/{row['source_id']}").decode()})
|
|
||||||
listing = module.parse
|
listing = module.parse
|
||||||
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
|
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
|
||||||
for row in listing(body, slug)]
|
for row in listing(body, slug)]
|
||||||
@@ -345,70 +338,25 @@ def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
|
|||||||
|
|
||||||
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
|
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
|
||||||
assert by_id["fake:acme:1"] == ("", "", None)
|
assert by_id["fake:acme:1"] == ("", "", None)
|
||||||
# Read now, and its page stated neither: the listing's stand.
|
|
||||||
assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid")
|
|
||||||
# Never read: the listing's stand until its page is.
|
# Never read: the listing's stand until its page is.
|
||||||
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
|
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
|
||||||
|
|
||||||
|
|
||||||
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
|
def test_ids_no_longer_listed_leave_the_detail_lists(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
||||||
base = "https://jobs.fake.example/acme"
|
|
||||||
ids = [str(n) for n in range(5)]
|
|
||||||
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
|
|
||||||
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
|
|
||||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
|
||||||
module = _fake_host(detail=_detail)
|
|
||||||
items, state = _run(monkeypatch, module)
|
|
||||||
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
|
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
|
|
||||||
|
|
||||||
web["asked"].clear()
|
|
||||||
items, state = _run(monkeypatch, module, state)
|
|
||||||
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
|
|
||||||
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
|
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
||||||
base = "https://jobs.fake.example/acme"
|
|
||||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
|
||||||
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
|
|
||||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
|
||||||
assert [(i.source_id, i.body) for i in items] == [
|
|
||||||
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
|
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
|
|
||||||
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
|
|
||||||
assert state["ats:complete:fake:acme"] == "1"
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
|
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
||||||
base = "https://jobs.fake.example/acme"
|
|
||||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
|
||||||
f"{base}/2": _status(429), f"{base}/3": b"three"})
|
|
||||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
|
||||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
|
||||||
assert [i.body for i in items] == ["one", "", ""]
|
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
|
||||||
assert state["ats:complete:fake:acme"] == "1"
|
|
||||||
|
|
||||||
|
|
||||||
def test_ids_no_longer_listed_leave_the_detailed_list(
|
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]),
|
||||||
|
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
||||||
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||||
|
|
||||||
|
|
||||||
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
"""A listing cut short says nothing about the roles it left out, so their
|
"""A listing cut short says nothing about the roles it left out, so their
|
||||||
ids stay, and the next pull does not read them in detail again."""
|
ids stay, and they are not read on their pages again."""
|
||||||
base = "https://jobs.fake.example/acme"
|
|
||||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
|
||||||
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
|
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
|
||||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
|
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
|
||||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||||
@@ -417,6 +365,19 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
|||||||
assert state["ats:complete:fake:acme"] == ""
|
assert state["ats:complete:fake:acme"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
||||||
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||||
|
source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
|
||||||
|
roles: Any = source.fetch() # a generator, which `close` stops
|
||||||
|
assert next(roles).body == ""
|
||||||
|
roles.close()
|
||||||
|
assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||||
|
|
||||||
|
|
||||||
def test_fetch_board_reads_the_listing_only(
|
def test_fetch_board_reads_the_listing_only(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
@@ -435,63 +396,137 @@ def test_a_host_without_detail_keeps_the_listings_description(
|
|||||||
assert "ats:detailed:fake:acme" not in state
|
assert "ats:detailed:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
def test_a_detail_answering_404_or_410_counts_as_read(
|
# -------------------------------------------------- read_board_details
|
||||||
|
|
||||||
|
NOW = datetime(2026, 9, 30, 12, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
||||||
|
state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]:
|
||||||
|
"""`ats.read_board_details` over the fake host's roles `ids`, stored
|
||||||
|
without text, in that order."""
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||||
|
rows = [row | {"description": ""}
|
||||||
|
for row in module.parse(json.dumps(ids).encode(), "acme")]
|
||||||
|
return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type]
|
||||||
|
now=kwargs.pop("now", NOW), **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_details_are_read_in_the_order_given_for_unread_roles_only(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
|
web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full",
|
||||||
f"{base}/3": b"three"}) # 1 answers 404
|
f"{base}/3": b"three in full"})
|
||||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
||||||
assert [i.body for i in items] == ["", "", "three"]
|
"ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})}
|
||||||
|
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"])
|
||||||
|
|
||||||
|
assert web["asked"] == [f"{base}/3", f"{base}/1"]
|
||||||
|
assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [
|
||||||
|
("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")]
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
|
||||||
|
|
||||||
|
def test_details_stop_at_the_cap_and_the_rest_wait(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
ids = [str(n) for n in range(5)]
|
||||||
|
web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids})
|
||||||
|
module = _fake_host(detail=_detail)
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
filled = _details(monkeypatch, module, state, ids, cap=2)
|
||||||
|
assert [r["description"] for r in filled] == ["full 0", "full 1"]
|
||||||
|
|
||||||
|
web["asked"].clear()
|
||||||
|
filled = _details(monkeypatch, module, state, ids[2:], cap=2)
|
||||||
|
assert web["asked"] == [f"{base}/2", f"{base}/3"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"),
|
||||||
|
f"{base}/3": _status(429), f"{base}/4": b"four"})
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"])
|
||||||
|
assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"]
|
||||||
|
assert [r["source_id"] for r in filled] == ["1"]
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||||
|
# A 429 is not the role's fault.
|
||||||
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
|
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
|
||||||
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
|
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||||
"""Two roles that always fail and a cap of two: the third is read on the
|
"""Roles never tried go before those that failed, within the cap."""
|
||||||
second pull, since roles never tried go before those that failed."""
|
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
|
web["answers"].update({f"{base}/1": failure, f"{base}/2": failure,
|
||||||
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
|
f"{base}/3": b"three", f"{base}/4": b"four"})
|
||||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
|
||||||
module = _fake_host(detail=_detail)
|
module = _fake_host(detail=_detail)
|
||||||
_items, state = _run(monkeypatch, module)
|
state: dict[str, str] = {}
|
||||||
|
_details(monkeypatch, module, state, ["1", "2"])
|
||||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||||
|
|
||||||
web["asked"].clear()
|
web["asked"].clear()
|
||||||
items, state = _run(monkeypatch, module, state)
|
_details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2)
|
||||||
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
|
assert web["asked"] == [f"{base}/3", f"{base}/4"]
|
||||||
assert [i.body for i in items] == ["", "", "three", "four"]
|
|
||||||
|
|
||||||
# With nothing new left, the failed ones are tried again, within the cap.
|
|
||||||
web["asked"].clear()
|
web["asked"].clear()
|
||||||
web["answers"][f"{base}/1"] = b"one at last"
|
web["answers"][f"{base}/1"] = b"one at last"
|
||||||
items, state = _run(monkeypatch, module, state)
|
filled = _details(monkeypatch, module, state, ["1", "2"], cap=2)
|
||||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
assert [r["description"] for r in filled] == ["one at last"]
|
||||||
assert [i.body for i in items] == ["one at last", "", "", ""]
|
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||||
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
|
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
|
||||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||||
|
|
||||||
|
|
||||||
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
|
def test_a_detail_answering_404_or_410_counts_as_read(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
|
web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404
|
||||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
state: dict[str, str] = {}
|
||||||
|
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"])
|
||||||
|
assert [r["source_id"] for r in filled] == ["3"]
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||||
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_failed_ids_no_longer_listed_leave_the_failed_list(
|
def test_a_read_without_text_is_read_again_after_some_days(
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
"""Finding 18: an Oracle requisition not yet published has an empty page
|
||||||
|
today and a full one next week."""
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
|
web["answers"][f"{base}/1"] = b""
|
||||||
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
module = _fake_host(detail=lambda get, slug, row: {
|
||||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
"description": get(f"{base}/{row['source_id']}").decode()})
|
||||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
state: dict[str, str] = {}
|
||||||
|
assert _details(monkeypatch, module, state, ["1"]) == []
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||||
|
assert json.loads(state["ats:detailempty:fake:acme"]) == {
|
||||||
|
"fake:acme:1": NOW.isoformat(timespec="seconds")}
|
||||||
|
|
||||||
|
web["asked"].clear()
|
||||||
|
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1)
|
||||||
|
assert _details(monkeypatch, module, state, ["1"], now=later) == []
|
||||||
|
assert web["asked"] == []
|
||||||
|
|
||||||
|
web["answers"][f"{base}/1"] = b"published now"
|
||||||
|
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS)
|
||||||
|
[filled] = _details(monkeypatch, module, state, ["1"], now=later)
|
||||||
|
assert filled["description"] == "published now"
|
||||||
|
assert json.loads(state["ats:detailempty:fake:acme"]) == {}
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_read_before_the_dates_were_kept_is_read_again(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
web["answers"]["https://jobs.fake.example/acme/1"] = b"one"
|
||||||
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
|
||||||
|
[filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"])
|
||||||
|
assert filled["description"] == "one"
|
||||||
|
|
||||||
|
|
||||||
def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||||
@@ -499,9 +534,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
|||||||
caplog: pytest.LogCaptureFixture) -> None:
|
caplog: pytest.LogCaptureFixture) -> None:
|
||||||
"""`salary_text`, `department` and `title` would be rewritten from the
|
"""`salary_text`, `department` and `title` would be rewritten from the
|
||||||
listing on the next pull, so detail does not fill them."""
|
listing on the next pull, so detail does not fill them."""
|
||||||
base = "https://jobs.fake.example/acme"
|
|
||||||
web["answers"].update({f"{base}/list": b'["1"]'})
|
|
||||||
|
|
||||||
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||||
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
|
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
|
||||||
"salary_min": 60000, "salary_max": 70000,
|
"salary_min": 60000, "salary_max": 70000,
|
||||||
@@ -510,7 +542,8 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
|||||||
"title": "Another title"}
|
"title": "Another title"}
|
||||||
|
|
||||||
caplog.set_level("DEBUG", logger=_runner.log.name)
|
caplog.set_level("DEBUG", logger=_runner.log.name)
|
||||||
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
[row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"])
|
||||||
|
item = ats.item_of(row, "fake:acme", "Acme")
|
||||||
assert (item.body, item.title, item.published_at) == (
|
assert (item.body, item.title, item.published_at) == (
|
||||||
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
||||||
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
|
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
|
||||||
@@ -522,66 +555,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
|||||||
"salary_max", "posted"}
|
"salary_max", "posted"}
|
||||||
|
|
||||||
|
|
||||||
def _paged_host() -> types.ModuleType:
|
|
||||||
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
|
|
||||||
module: Any = _fake_host(detail=_detail)
|
|
||||||
del module.feed_url, module.parse
|
|
||||||
|
|
||||||
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
|
|
||||||
index = int(cursor or 0)
|
|
||||||
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
|
|
||||||
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
|
|
||||||
|
|
||||||
module.page = page
|
|
||||||
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
|
|
||||||
assert not readers.problems(module, "fake")
|
|
||||||
return module
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_paged_host_reads_every_page_then_each_role_once(
|
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
||||||
base = "https://jobs.fake.example/acme"
|
|
||||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
|
|
||||||
f"{base}/1": b"one", f"{base}/2": b"two"})
|
|
||||||
items, state = _run(monkeypatch, _paged_host())
|
|
||||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
|
|
||||||
f"{base}/1", f"{base}/2"]
|
|
||||||
assert [i.body for i in items] == ["one", "two"]
|
|
||||||
assert state["ats:complete:fake:acme"] == "1"
|
|
||||||
|
|
||||||
web["asked"].clear()
|
|
||||||
items, state = _run(monkeypatch, _paged_host(), state)
|
|
||||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
|
|
||||||
assert [i.body for i in items] == ["", ""]
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
|
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
||||||
base = "https://jobs.fake.example/acme"
|
|
||||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
|
|
||||||
f"{base}/1": b"one"})
|
|
||||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
|
||||||
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
|
|
||||||
_items, state = _run(monkeypatch, _paged_host(), state)
|
|
||||||
assert state["ats:complete:fake:acme"] == ""
|
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
|
||||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
|
||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
||||||
"""The consumer stops after the first role, so the second, read but not
|
|
||||||
stored, is read again on the next pull."""
|
|
||||||
base = "https://jobs.fake.example/acme"
|
|
||||||
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
|
|
||||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
|
||||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
|
||||||
roles: Any = source.fetch() # a generator, which `close` stops
|
|
||||||
assert next(roles).body == "one"
|
|
||||||
roles.close()
|
|
||||||
assert "ats:detailed:fake:acme" not in source.state
|
|
||||||
|
|
||||||
|
|
||||||
# ----------------------------------------------------------------- Lever EU
|
# ----------------------------------------------------------------- Lever EU
|
||||||
|
|
||||||
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||||
|
|||||||
+11
-9
@@ -1053,7 +1053,9 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
|||||||
assert batch is not None
|
assert batch is not None
|
||||||
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
||||||
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
||||||
assert stages == list(cascade.CASCADES["pull"])
|
# Enrich found no board with role pages, so the embed and score behind it
|
||||||
|
# are passed over.
|
||||||
|
assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"]
|
||||||
work = [j.kind for j in jobs if j.kind != FINAL]
|
work = [j.kind for j in jobs if j.kind != FINAL]
|
||||||
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
||||||
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
||||||
@@ -1107,10 +1109,10 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
|
|||||||
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
||||||
"""By position, not by name: `score` is in the chain twice, once before
|
"""By position, not by name: `score` is in the chain twice, once before
|
||||||
extract and once after, and the second must lead on to summarise."""
|
extract and once after, and the second must lead on to summarise."""
|
||||||
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
|
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score",
|
||||||
"summarise")
|
"extract", "score", "summarise")
|
||||||
assert [cascade.follows("pull", at) for at in range(6)] == \
|
assert [cascade.follows("pull", at) for at in range(9)] == \
|
||||||
["embed", "score", "extract", "score", "summarise", ""]
|
["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""]
|
||||||
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
|
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
|
||||||
assert cascade.follows("nope", 0) == ""
|
assert cascade.follows("nope", 0) == ""
|
||||||
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
||||||
@@ -1124,10 +1126,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||||||
made.append(t)
|
made.append(t)
|
||||||
return t
|
return t
|
||||||
|
|
||||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
|
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5},
|
||||||
context=Context())
|
context=Context())
|
||||||
assert out == {"task": "score", "next": "extract"}
|
assert out == {"task": "score", "next": "extract"}
|
||||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
|
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6}
|
||||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
|
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
|
||||||
context=Context())
|
context=Context())
|
||||||
assert out["next"] == "summarise" and made[1].carries["at"] == 5
|
assert out["next"] == "summarise" and made[1].carries["at"] == 8
|
||||||
Reference in new issue
Block a user