Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:29:19 +02:00
1 parent 07e3bb4756
commit b148647ac3
5 files changed
+346 -40

No files matched your search

+51 -31
View File
@@ -2,18 +2,22 @@
Some hosts list a role without its text and state it on the role's own page
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
with a `detail`). The pull reads listings only; this stage reads those pages
after scoring, so the calls go where the ranking is:
with a `detail`, and a search provider such as Jobbank). The pull reads
listings only; this stage reads those pages after scoring, so the calls go
where the ranking is. A saved search whose reader has `detail` is treated as
a board under its target `provider:name`: the same bar, budgets and state
keys, and its failing record is `search:failing:<target>`.
1. `enrich_waiting` hands over the open roles without text that an enabled
agent with a prompt ranks ungated among its best `enrich.top` and at
`enrich.min_score` or above, best first by the highest such score. A role
below that bar for every agent gets no call.
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
out the roles already read, unless their read failed or gave no text
`enrich.retry_days` ago (`plugins.ats.detail_due`).
3. One batch per board, on the `http` pool, reads them
2. They are taken in that order, those whose read failed last, on boards and
searches whose reader has `detail`, at most `enrich.per_board` from one
board and `enrich.per_run` in all, leaving out the roles already read,
unless their read failed or gave no text `enrich.retry_days` ago
(`plugins.ats.detail_due`).
3. One batch per board or search, on the `http` pool, reads them
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
what each page filled. `Store.upsert_role` drops the vector of a role
@@ -37,6 +41,7 @@ import httpx
from app.config import Config
from app.jobs import stages
from app.models import Subscription
from app.storage import Store
from pipeline import NULL, Batch, RunContext, Tally, Task, job
@@ -53,7 +58,14 @@ def _ats() -> Any:
return ats
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
def _native(found: dict[str, Any], sub: Subscription) -> str:
"""The host's own id of a stored role: a board's roles are stored as
`host:slug:id`, a search's as `provider:id`."""
prefix = sub.target if sub.source == "ats" else sub.target.partition(":")[0]
return str(found["source_id"]).removeprefix(f"{prefix}:")
def _row(found: dict[str, Any], sub: Subscription) -> dict[str, Any]:
"""A stored role back as the row its reader listed: the stored `raw`
with the host's own id, the title, the link and the date."""
try:
@@ -65,29 +77,36 @@ def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
except ValueError:
posted = None
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
return raw | {"source_id": _native(found, sub),
"title": found["title"], "url": found["url"], "description": "",
"posted": posted}
def _readable(store: Store) -> dict[int, Subscription]:
"""The fetchable boards and searches whose reader has `detail`, by
subscription id."""
ats = _ats()
return {sub.id: sub for source in ("ats", "search")
for sub in store.subscriptions(source=source, fetchable=True)
if hasattr(ats.reader_of(sub.target), "detail")}
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
"""`(target, uids)` per board, best board first, within the budgets: the
roles `enrich_waiting` ranks, on fetchable boards whose reader has
`detail` and that are not failing, and due for a read."""
"""`(target, uids)` per board or search, best first, within the budgets:
the roles `enrich_waiting` ranks, on fetchable boards and searches whose
reader has `detail` and that are not failing, and due for a read."""
settings = config.enrich
ats = _ats()
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
subs = _readable(store)
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
for found in store.enrich_waiting(settings.top, settings.min_score):
target = boards.get(found["subscription_id"])
if target is None:
sub = subs.get(found["subscription_id"])
if sub is None or f"{sub.source}:failing:{sub.target}" in store.state:
continue
module = ats.readers.HOSTS.get(target.partition(":")[0])
if module is None or not hasattr(module, "detail") \
or f"ats:failing:{target}" in store.state:
continue
by_board[target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
# The id the board's per-role state keeps: `host:slug:id`.
found["compound"] = f"{sub.target}:{_native(found, sub)}"
by_board[sub.target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["compound"] for f in found],
retry_days=settings.retry_days))
for target, found in by_board.items()}
# Best first; roles whose read failed after every other, so a page that
@@ -95,13 +114,13 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
for target in by_board}
ranked = sorted((f for found in by_board.values() for f in found),
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
key=lambda f: (f["compound"] in failed[subs[f["subscription_id"]].target],
-float(f["score"] or 0), f["uid"]))
picked: dict[str, list[str]] = {}
total = 0
for found in ranked:
target = boards[found["subscription_id"]]
if found["source_id"] not in due[target]:
target = subs[found["subscription_id"]].target
if found["compound"] not in due[target]:
continue
if total >= settings.per_run:
break
@@ -114,7 +133,8 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
def batches(config: Config, store: Store) -> list[Batch]:
"""The finder: one batch per board, nothing while the stage is off."""
"""The finder: one batch per board or search, nothing while the stage is
off."""
if not stages.stage_on(config, store, "enrich"):
return []
return [Batch({"target": target, "uids": uids},
@@ -125,16 +145,16 @@ def batches(config: Config, store: Store) -> list[Batch]:
@job(KIND, "Reading roles on their own pages", resource="http")
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Read one board's roles in `uids` on their own pages, in that order,
and store what each page filled. `filled` is how many roles changed,
`texts` how many of them now have text."""
"""Read one board's or search's roles in `uids` on their own pages, in
that order, and store what each page filled. `filled` is how many roles
changed, `texts` how many of them now have text."""
ats = _ats()
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
subs = [s for s in _readable(store).values() if s.target == target]
if not subs or not uids:
return {"target": target, "filled": 0, "texts": 0}
sub = subs[0]
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
rows = [_row(found[uid], target) for uid in uids if uid in found]
rows = [_row(found[uid], sub) for uid in uids if uid in found]
context.check()
with httpx.Client() as client:
filled = ats.read_board_details(store.state, target, rows, client=client,
@@ -143,7 +163,7 @@ def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
texts = 0
for row in filled:
item = ats.item_of(row, target, label)
item.source = "ats"
item.source = sub.source
store.upsert_role(item, sub)
texts += bool(row.get("description"))
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
+51 -2
View File
@@ -5,6 +5,11 @@ link, keep the text as `article_text`. It is on demand, never in the pull,
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
and one click for a posting somebody opened is fair.
A role of a search whose reader has `detail` (Jobbank) is read through that
`detail`, which reads the ad's own page and states its text, rather than
scraping the link; a failure of that read is an `Unreadable` with the
category its HTTP status or exception names.
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
ad and a "Se jobbet" button to the posting itself: the employer's site, The
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
@@ -19,9 +24,14 @@ message the route answers with.
from __future__ import annotations
from datetime import datetime
from typing import Any
import httpx
from app.models import Role
from app.storage import Store
from desk.tools import scrape
from desk.tools import failures, scrape
from desk.tools.checks import Page
from plugins.readers import jobindex
@@ -113,6 +123,43 @@ def _jobindex_text(store: Store, role: Role) -> str:
raise Unreadable(failure or own.failure)
def _detail_target(role: Role) -> str:
"""The search target of a role whose reader has `detail`, else ''."""
from plugins import ats
target = str(role.raw.get("search") or "")
if role.source == "search" and hasattr(ats.reader_of(target), "detail"):
return target
return ""
def _detail_text(role: Role, target: str) -> str:
"""The text of a search role, read by its reader's `detail` on the ad's
own page as the pull's enrich stage reads it. Raises `Unreadable` when
the read fails or the page states no text."""
from plugins import ats
provider = target.partition(":")[0]
try:
posted: datetime | None = datetime.fromisoformat(role.published_at)
except ValueError:
posted = None
row: dict[str, Any] = role.raw | {
"source_id": role.source_id.removeprefix(f"{provider}:"), "title": role.title,
"url": role.original_url, "description": "", "posted": posted}
try:
with httpx.Client() as client:
found = ats.read_detail(client, target, row)
except httpx.HTTPStatusError as exc:
raise Unreadable(failures.of_status(exc.response.status_code)) from exc
except Exception as exc:
category = failures.of_exception(exc)
raise Unreadable("" if category == failures.UNKNOWN else category) from exc
if not (text := str(found.get("description") or "").strip()):
raise Unreadable("empty")
return text
def fetch_posting(store: Store, uid: str) -> Role:
"""Fetch the posting page once and keep it as plain text, for a role
whose description is a teaser. Synchronous: one request the person asked
@@ -124,7 +171,9 @@ def fetch_posting(store: Store, uid: str) -> Role:
raise AlreadyFetched("The posting text is already fetched.")
if not role.original_url:
raise NoLink("This role has no link to fetch the posting from.")
if _is_jobindex(role):
if target := _detail_target(role):
text = _detail_text(role, target)
elif _is_jobindex(role):
text = _jobindex_text(store, role)
else:
got = scrape.scrape(role.original_url)
+46 -7
View File
@@ -38,7 +38,9 @@ that read once it is complete, rather than by a second request.
A pull reads listings only. A host that states a role's text on the role's
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
scoring, best roles first, through `read_board_details`.
scoring, best roles first, through `read_board_details`. A search provider
whose reader has `detail` (Jobbank) is read the same way, under its search
target `provider:name`; `read_detail` is the one read the Fetch button makes.
The limits below are read when a board is read, so a test that replaces one
on this module reaches the runner.
@@ -53,6 +55,7 @@ import sys
from collections.abc import Callable, Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import ModuleType
from typing import Any
import httpx
@@ -114,11 +117,18 @@ def _limits(stale_days: int | None = None) -> _runner.Limits:
stale_days=stale_days)
def reader_of(target: str) -> ModuleType | None:
"""The reader of a board (`host:slug`) or of a search (`provider:name`),
None for a name that is neither. The two never share a name."""
host = target.partition(":")[0]
return readers.HOSTS.get(host) or readers.PROVIDERS.get(host)
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
"""The hosts a request or a redirect may land on for one board: what its
host module's `allowed` says. Anything else is refused, at the first hop
and at every redirect."""
module = readers.HOSTS.get(host)
"""The hosts a request or a redirect may land on for one board or search:
what its module's `allowed` says. Anything else is refused, at the first
hop and at every redirect."""
module = reader_of(host)
return module.allowed(slug) if module is not None else lambda h: False
@@ -540,7 +550,9 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
are written and changed by what this read did, so a pull that dropped ids
meanwhile keeps its drop."""
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
module = reader_of(target)
if module is None or not hasattr(module, "detail"):
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
keys = detail_keys(target)
now = now or datetime.now(UTC)
@@ -582,12 +594,39 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
return filled
def read_detail(client: httpx.Client, target: str, row: Row) -> dict[str, Any]:
"""One role of board or search `target` read on its own page: what the
reader's `detail` returns, for the Fetch button. No state is kept and no
field is filtered. Raises what `detail` raises, and ValueError for a
target whose reader has none."""
host, _, slug = target.partition(":")
module = reader_of(target)
if module is None or not hasattr(module, "detail"):
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
return module.detail(_getter(client, host, slug), slug, row)
def item_of(row: Row, target: str, label: str) -> Item:
"""One row of board `target` as the `Item` the store takes: its id
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
date is dated now, as `Item` itself would date it."""
date is dated now, as `Item` itself would date it. A search's row is
stored as `plugins/search.py` stores it: `provider:id`, the employer from
`company` and `label` the search's own."""
host, _, slug = target.partition(":")
description = row.get("description") or ""
if host not in readers.HOSTS:
return Item(
source_id=f"{host}:{row['source_id']}",
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted") or datetime.now(UTC),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {"company": row.get("company") or "", "provider": host,
"search": target},
)
return Item(
source_id=f"{host}:{slug}:{row['source_id']}",
location=row["url"],
+116
View File
@@ -239,6 +239,122 @@ def test_a_board_switched_off_since_the_plan_reads_nothing(
assert web["asked"] == []
# ----------------------------------------------------------- search roles
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
AD_URL = "https://jobbank.dk/job/1234/acme/role"
#: `Store.enrich_waiting` hands over `source = 'ats'` roles only, and a search's
#: are `search`: until it takes both, the tests that need its answer fail.
needs_waiting = pytest.mark.xfail(
reason="Store.enrich_waiting reads source 'ats' only", strict=False)
def search(store: Store, target: str = "jobbank:cph") -> Any:
return store.add_subscription("search", target, label=target, params={"q": "x"})
def search_role(store: Store, target: str, native: str, *, provider: str = "") -> str:
"""A role of search `target` as a pull stores it: `provider:id`, the
employer in `raw` and no text."""
provider = provider or target.partition(":")[0]
sub = next(s for s in store.subscriptions(source="search") if s.target == target)
row = {"source_id": native, "title": "Researcher", "url": AD_URL.replace("1234", native),
"description": "", "company": "Acme A/S", "places": [{"text": "Fyn", "remote": None}]}
item = ats.item_of(row, target, sub.label)
item.source = "search"
assert item.source_id == f"{provider}:{native}"
store.upsert_role(item, sub)
return item.uid
@needs_waiting
def test_a_search_whose_reader_has_detail_is_planned_like_a_board(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
search(store)
search(store, "jobindex:cph") # no `detail`: left out
board(store, "fake:acme")
a, b = search_role(store, "jobbank:cph", "1234"), search_role(store, "jobbank:cph", "99")
j = search_role(store, "jobindex:cph", "7")
f = role(store, "fake:acme", "1")
scored(store, "One", {a: 0.9, b: 0.6, j: 0.95, f: 0.7})
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [a, b]), ("fake:acme", [f])]
# The budgets are a board's: `per_board` and `per_run` count the search as one.
assert enrich.plan(config(tmp_path, per_board=1), store) == [("jobbank:cph", [a]),
("fake:acme", [f])]
assert enrich.plan(config(tmp_path, per_run=2), store) == [("jobbank:cph", [a]),
("fake:acme", [f])]
# A failing search is left out, as a failing board is.
store.state["search:failing:jobbank:cph"] = "{}"
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [f])]
@needs_waiting
def test_a_search_role_read_waits_for_the_retry_and_a_failed_one_goes_last(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
search(store)
one, two, three = (search_role(store, "jobbank:cph", n) for n in ("1", "2", "3"))
scored(store, "One", {one: 0.9, two: 0.8, three: 0.7})
store.state["ats:detailed:jobbank:cph"] = json.dumps(["jobbank:cph:1", "jobbank:cph:2"])
store.state["ats:detailfailed:jobbank:cph"] = json.dumps(["jobbank:cph:2"])
store.state["ats:detailempty:jobbank:cph"] = json.dumps(
{"jobbank:cph:1": datetime.now(UTC).isoformat(timespec="seconds"),
"jobbank:cph:2": "2020-01-01T00:00:00+00:00"})
# 1 gave no text a moment ago and waits; 2 failed and is due, after 3.
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [three, two])]
@needs_waiting
def test_a_batch_reads_a_search_roles_page_and_stores_its_text(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
search(store)
uid = search_role(store, "jobbank:cph", "1234")
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32))])
web["answers"][AD_URL] = AD
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[uid])
assert out == {"target": "jobbank:cph", "read": 1, "filled": 1, "texts": 1}
assert web["asked"] == [AD_URL]
got = store.role(uid)
assert got is not None and got.description
assert (got.source, got.source_id, got.company) == ("search", "jobbank:1234", "Acme A/S")
assert got.raw["search"] == "jobbank:cph" and got.raw["provider"] == "jobbank"
assert store.vectors("fake") == {}
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == ["jobbank:cph:1234"]
# Read, it is not planned again, and has no text left to wait for.
scored(store, "One", {uid: 0.9})
assert enrich.plan(config(tmp_path), store) == []
@needs_waiting
def test_a_search_batch_counts_a_404_as_read_as_a_board_does(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
search(store)
one, two = search_role(store, "jobbank:cph", "1"), search_role(store, "jobbank:cph", "2")
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[one, two])
assert out["filled"] == 0 and out["read"] == 2
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == [
"jobbank:cph:1", "jobbank:cph:2"]
def test_a_reader_is_found_for_a_board_and_a_search_and_one_without_detail_is_refused(
web: dict[str, Any]) -> None:
assert ats.reader_of("fake:acme") is readers.HOSTS["fake"]
assert ats.reader_of("jobbank:cph") is readers.PROVIDERS["jobbank"]
assert ats.reader_of("nowhere:x") is None
with httpx.Client() as client:
for target in ("jobindex:cph", "plain:acme", "nowhere:x"):
with pytest.raises(ValueError, match="no reader"):
ats.read_detail(client, target, {"source_id": "1", "url": BASE})
with pytest.raises(ValueError, match="no reader"):
ats.read_board_details({}, target, [{"source_id": "1", "url": BASE}],
client=client)
# ------------------------------------------------------------------ cascade
class _Context:
+82
View File
@@ -8,12 +8,15 @@ from pathlib import Path
import pytest
import httpx
from app.jobs import posting
from app.storage import Store
from desk.models import Item
from desk.tools import scrape
from desk.tools.article import ArticleContent
from desk.tools.checks import Page
from plugins import ats
from prepped import upsert
ROOT = Path(__file__).resolve().parents[1]
@@ -164,3 +167,82 @@ def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
web = Web(monkeypatch, {}, {LINK: "The real posting."})
assert posting.fetch_posting(store, uid).article_text == "The real posting."
assert web.asked == [LINK]
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
AD_URL = "https://jobbank.dk/job/1234/acme/role"
def search_role(store: Store, provider: str = "jobbank", url: str = AD_URL) -> str:
"""A role of a saved search as a pull stores it: `provider:id`, source
`search`, the search's target in `raw`."""
target = f"{provider}:cph"
sub = store.add_subscription("search", target, label=target, params={"q": "x"})
row = {"source_id": "1234", "title": "Researcher", "url": url, "description": "",
"company": "Acme A/S"}
item = ats.item_of(row, target, target)
item.source = "search"
upsert(store, item, sub)
return item.uid
class Pages:
"""`ats._get` answering from `answers` (URL -> body or the exception to
raise), remembering what it was asked."""
def __init__(self, monkeypatch: pytest.MonkeyPatch, answers: dict[str, object]) -> None:
self.asked: list[str] = []
self.answers = answers
monkeypatch.setattr(ats, "_get", self.get)
def get(self, client: object, url: str, host: str, slug: str, **kw: object) -> bytes:
self.asked.append(url)
answer = self.answers[url]
if isinstance(answer, Exception):
raise answer
return answer # type: ignore[return-value]
def test_a_search_role_whose_reader_has_detail_is_read_by_its_detail(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = search_role(store)
pages = Pages(monkeypatch, {AD_URL: AD})
web = Web(monkeypatch, {}, {})
got = posting.fetch_posting(store, uid)
assert got.article_text and got.article_at and got.posting_url == ""
assert pages.asked == [AD_URL] and web.asked == [] # no scrape of the link
with pytest.raises(posting.AlreadyFetched):
posting.fetch_posting(store, uid)
@pytest.mark.parametrize(("answer", "message"), [
(httpx.HTTPStatusError("404", request=httpx.Request("GET", AD_URL),
response=httpx.Response(404, request=httpx.Request("GET", AD_URL))),
"gone"),
(httpx.HTTPStatusError("429", request=httpx.Request("GET", AD_URL),
response=httpx.Response(429, request=httpx.Request("GET", AD_URL))),
"refusing"),
(httpx.ConnectError("down"), "could not be reached"),
(b"<html>Just a moment...</html>", "Nothing readable"),
(b'<script type="application/ld+json">{"@type": "JobPosting"}</script>', "no readable text"),
])
def test_a_failed_detail_read_says_why(store: Store, monkeypatch: pytest.MonkeyPatch,
answer: object, message: str) -> None:
uid = search_role(store)
Pages(monkeypatch, {AD_URL: answer})
with pytest.raises(posting.Unreadable, match=message) as failure:
posting.fetch_posting(store, uid)
assert failure.value.status == 422
stored = store.role(uid)
assert stored is not None and stored.article_text == ""
def test_a_search_role_without_detail_is_scraped_from_its_link(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = search_role(store, "adzuna", "https://www.adzuna.dk/details/1")
pages = Pages(monkeypatch, {})
Web(monkeypatch, {}, {"https://www.adzuna.dk/details/1": "The posting."})
assert posting.fetch_posting(store, uid).article_text == "The posting."
assert pages.asked == []