Read role pages in an enrich stage after scoring, best roles first

The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open
roles without text that an enabled agent ranks ungated in its best
`enrich.top` at `enrich.min_score` or above, best score first, on boards
whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`,
one batch per board. `plugins.ats.read_board_details` reads them and keeps
the per-board state; a read that gave no text is read again after
`enrich.retry_days` (finding 18). Filled roles are upserted, which drops
their vectors, and embed and score follow; `cascade.step` passes over both
when nothing was filled.

The pull cascade is pull, embed, score, enrich, embed, score, extract, score,
summarise. Saving an agent and changing a company's affinity queue the
`enrich` job (enrich, embed, score) behind their rescore; a vote does not.
The stage has a switch, `stage:enrich`, over `enrich.enabled`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:53:29 +02:00
1 parent 3adafd8950
commit 77b45a4b0f
10 files changed
+956 -219

No files matched your search

+136 -163
View File
@@ -1,6 +1,7 @@
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
every host: the registry and its contract check, the fetch a host is handed,
the per-role detail reads, and Lever's EU instance. No network —
the per-role detail reads the `enrich` stage makes, and Lever's EU
instance. No network —
a fake host stands in where a behaviour belongs to no real host yet."""
from __future__ import annotations
@@ -11,7 +12,7 @@ import subprocess
import sys
import types
from collections.abc import Callable
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -308,34 +309,26 @@ def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
# ---------------------------------------------------------------- detail
def test_detail_is_read_for_unread_roles_only(
def test_a_pull_reads_no_role_page(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The pages are `enrich`'s: the pull reads the listing only, and sends
every row of a host with `detail` without a description."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
module = _fake_host(detail=_detail)
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
# What detail returns replaces the listing's field; a role not read keeps the listing's.
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert web["asked"] == [f"{base}/list"]
assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")]
assert "ats:detailed:fake:acme" not in state
assert state["ats:complete:fake:acme"] == "1"
def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
def test_a_role_read_on_its_page_keeps_the_place_its_page_gave(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The listing states a place on every pull, and a filled value replaces
a filled column, so for a role already read it is withheld."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full",
f"{base}/3": RuntimeError("down")})
module = _fake_host(detail=lambda get, slug, row: {
"description": get(f"{base}/{row['source_id']}").decode()})
web["answers"].update({f"{base}/list": b'["1", "3"]'})
module = _fake_host(detail=_detail)
listing = module.parse
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
for row in listing(body, slug)]
@@ -345,70 +338,25 @@ def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
assert by_id["fake:acme:1"] == ("", "", None)
# Read now, and its page stated neither: the listing's stand.
assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid")
# Never read: the listing's stand until its page is.
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
ids = [str(n) for n in range(5)]
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
module = _fake_host(detail=_detail)
items, state = _run(monkeypatch, module)
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
assert state["ats:complete:fake:acme"] == "1"
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": _status(429), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
assert state["ats:complete:fake:acme"] == "1"
def test_ids_no_longer_listed_leave_the_detailed_list(
def test_ids_no_longer_listed_leave_the_detail_lists(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""A listing cut short says nothing about the roles it left out, so their
ids stay, and the next pull does not read them in detail again."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
ids stay, and they are not read on their pages again."""
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
@@ -417,6 +365,19 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
assert state["ats:complete:fake:acme"] == ""
def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
roles: Any = source.fetch() # a generator, which `close` stops
assert next(roles).body == ""
roles.close()
assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
def test_fetch_board_reads_the_listing_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
@@ -435,63 +396,137 @@ def test_a_host_without_detail_keeps_the_listings_description(
assert "ats:detailed:fake:acme" not in state
def test_a_detail_answering_404_or_410_counts_as_read(
# -------------------------------------------------- read_board_details
NOW = datetime(2026, 9, 30, 12, tzinfo=UTC)
def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]:
"""`ats.read_board_details` over the fake host's roles `ids`, stored
without text, in that order."""
monkeypatch.setitem(readers.HOSTS, "fake", module)
rows = [row | {"description": ""}
for row in module.parse(json.dumps(ids).encode(), "acme")]
return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type]
now=kwargs.pop("now", NOW), **kwargs)
def test_details_are_read_in_the_order_given_for_unread_roles_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
f"{base}/3": b"three"}) # 1 answers 404
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [i.body for i in items] == ["", "", "three"]
web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full",
f"{base}/3": b"three in full"})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
"ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"])
assert web["asked"] == [f"{base}/3", f"{base}/1"]
assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [
("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
def test_details_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
ids = [str(n) for n in range(5)]
web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids})
module = _fake_host(detail=_detail)
state: dict[str, str] = {}
filled = _details(monkeypatch, module, state, ids, cap=2)
assert [r["description"] for r in filled] == ["full 0", "full 1"]
web["asked"].clear()
filled = _details(monkeypatch, module, state, ids[2:], cap=2)
assert web["asked"] == [f"{base}/2", f"{base}/3"]
def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"),
f"{base}/3": _status(429), f"{base}/4": b"four"})
state: dict[str, str] = {}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"])
assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"]
assert [r["source_id"] for r in filled] == ["1"]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
# A 429 is not the role's fault.
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
"""Two roles that always fail and a cap of two: the third is read on the
second pull, since roles never tried go before those that failed."""
"""Roles never tried go before those that failed, within the cap."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
web["answers"].update({f"{base}/1": failure, f"{base}/2": failure,
f"{base}/3": b"three", f"{base}/4": b"four"})
module = _fake_host(detail=_detail)
_items, state = _run(monkeypatch, module)
state: dict[str, str] = {}
_details(monkeypatch, module, state, ["1", "2"])
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
assert [i.body for i in items] == ["", "", "three", "four"]
_details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2)
assert web["asked"] == [f"{base}/3", f"{base}/4"]
# With nothing new left, the failed ones are tried again, within the cap.
web["asked"].clear()
web["answers"][f"{base}/1"] = b"one at last"
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one at last", "", "", ""]
filled = _details(monkeypatch, module, state, ["1", "2"], cap=2)
assert [r["description"] for r in filled] == ["one at last"]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
def test_a_detail_answering_404_or_410_counts_as_read(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404
state: dict[str, str] = {}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"])
assert [r["source_id"] for r in filled] == ["3"]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
assert json.loads(state["ats:detailed:fake:acme"]) == []
def test_failed_ids_no_longer_listed_leave_the_failed_list(
def test_a_read_without_text_is_read_again_after_some_days(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""Finding 18: an Oracle requisition not yet published has an empty page
today and a full one next week."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
web["answers"][f"{base}/1"] = b""
module = _fake_host(detail=lambda get, slug, row: {
"description": get(f"{base}/{row['source_id']}").decode()})
state: dict[str, str] = {}
assert _details(monkeypatch, module, state, ["1"]) == []
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
assert json.loads(state["ats:detailempty:fake:acme"]) == {
"fake:acme:1": NOW.isoformat(timespec="seconds")}
web["asked"].clear()
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1)
assert _details(monkeypatch, module, state, ["1"], now=later) == []
assert web["asked"] == []
web["answers"][f"{base}/1"] = b"published now"
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS)
[filled] = _details(monkeypatch, module, state, ["1"], now=later)
assert filled["description"] == "published now"
assert json.loads(state["ats:detailempty:fake:acme"]) == {}
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
def test_a_role_read_before_the_dates_were_kept_is_read_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"]["https://jobs.fake.example/acme/1"] = b"one"
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
[filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"])
assert filled["description"] == "one"
def test_detail_fills_only_the_fields_a_later_pull_keeps(
@@ -499,9 +534,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
caplog: pytest.LogCaptureFixture) -> None:
"""`salary_text`, `department` and `title` would be rewritten from the
listing on the next pull, so detail does not fill them."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]'})
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
"salary_min": 60000, "salary_max": 70000,
@@ -510,7 +542,8 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
"title": "Another title"}
caplog.set_level("DEBUG", logger=_runner.log.name)
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
[row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"])
item = ats.item_of(row, "fake:acme", "Acme")
assert (item.body, item.title, item.published_at) == (
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
@@ -522,66 +555,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
"salary_max", "posted"}
def _paged_host() -> types.ModuleType:
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
module: Any = _fake_host(detail=_detail)
del module.feed_url, module.parse
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
index = int(cursor or 0)
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
module.page = page
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
assert not readers.problems(module, "fake")
return module
def test_a_paged_host_reads_every_page_then_each_role_once(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
f"{base}/1": b"one", f"{base}/2": b"two"})
items, state = _run(monkeypatch, _paged_host())
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "two"]
assert state["ats:complete:fake:acme"] == "1"
web["asked"].clear()
items, state = _run(monkeypatch, _paged_host(), state)
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
assert [i.body for i in items] == ["", ""]
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
f"{base}/1": b"one"})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
_items, state = _run(monkeypatch, _paged_host(), state)
assert state["ats:complete:fake:acme"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The consumer stops after the first role, so the second, read but not
stored, is read again on the next pull."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
roles: Any = source.fetch() # a generator, which `close` stops
assert next(roles).body == "one"
roles.close()
assert "ats:detailed:fake:acme" not in source.state
# ----------------------------------------------------------------- Lever EU
def test_a_lever_slug_may_name_the_eu_instance() -> None: