Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:29:19 +02:00
1 parent 07e3bb4756
commit b148647ac3
5 files changed
+346 -40

No files matched your search

+116
View File
@@ -239,6 +239,122 @@ def test_a_board_switched_off_since_the_plan_reads_nothing(
assert web["asked"] == []
# ----------------------------------------------------------- search roles
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
AD_URL = "https://jobbank.dk/job/1234/acme/role"
#: `Store.enrich_waiting` hands over `source = 'ats'` roles only, and a search's
#: are `search`: until it takes both, the tests that need its answer fail.
needs_waiting = pytest.mark.xfail(
reason="Store.enrich_waiting reads source 'ats' only", strict=False)
def search(store: Store, target: str = "jobbank:cph") -> Any:
return store.add_subscription("search", target, label=target, params={"q": "x"})
def search_role(store: Store, target: str, native: str, *, provider: str = "") -> str:
"""A role of search `target` as a pull stores it: `provider:id`, the
employer in `raw` and no text."""
provider = provider or target.partition(":")[0]
sub = next(s for s in store.subscriptions(source="search") if s.target == target)
row = {"source_id": native, "title": "Researcher", "url": AD_URL.replace("1234", native),
"description": "", "company": "Acme A/S", "places": [{"text": "Fyn", "remote": None}]}
item = ats.item_of(row, target, sub.label)
item.source = "search"
assert item.source_id == f"{provider}:{native}"
store.upsert_role(item, sub)
return item.uid
@needs_waiting
def test_a_search_whose_reader_has_detail_is_planned_like_a_board(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
search(store)
search(store, "jobindex:cph") # no `detail`: left out
board(store, "fake:acme")
a, b = search_role(store, "jobbank:cph", "1234"), search_role(store, "jobbank:cph", "99")
j = search_role(store, "jobindex:cph", "7")
f = role(store, "fake:acme", "1")
scored(store, "One", {a: 0.9, b: 0.6, j: 0.95, f: 0.7})
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [a, b]), ("fake:acme", [f])]
# The budgets are a board's: `per_board` and `per_run` count the search as one.
assert enrich.plan(config(tmp_path, per_board=1), store) == [("jobbank:cph", [a]),
("fake:acme", [f])]
assert enrich.plan(config(tmp_path, per_run=2), store) == [("jobbank:cph", [a]),
("fake:acme", [f])]
# A failing search is left out, as a failing board is.
store.state["search:failing:jobbank:cph"] = "{}"
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [f])]
@needs_waiting
def test_a_search_role_read_waits_for_the_retry_and_a_failed_one_goes_last(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
search(store)
one, two, three = (search_role(store, "jobbank:cph", n) for n in ("1", "2", "3"))
scored(store, "One", {one: 0.9, two: 0.8, three: 0.7})
store.state["ats:detailed:jobbank:cph"] = json.dumps(["jobbank:cph:1", "jobbank:cph:2"])
store.state["ats:detailfailed:jobbank:cph"] = json.dumps(["jobbank:cph:2"])
store.state["ats:detailempty:jobbank:cph"] = json.dumps(
{"jobbank:cph:1": datetime.now(UTC).isoformat(timespec="seconds"),
"jobbank:cph:2": "2020-01-01T00:00:00+00:00"})
# 1 gave no text a moment ago and waits; 2 failed and is due, after 3.
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [three, two])]
@needs_waiting
def test_a_batch_reads_a_search_roles_page_and_stores_its_text(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
search(store)
uid = search_role(store, "jobbank:cph", "1234")
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32))])
web["answers"][AD_URL] = AD
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[uid])
assert out == {"target": "jobbank:cph", "read": 1, "filled": 1, "texts": 1}
assert web["asked"] == [AD_URL]
got = store.role(uid)
assert got is not None and got.description
assert (got.source, got.source_id, got.company) == ("search", "jobbank:1234", "Acme A/S")
assert got.raw["search"] == "jobbank:cph" and got.raw["provider"] == "jobbank"
assert store.vectors("fake") == {}
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == ["jobbank:cph:1234"]
# Read, it is not planned again, and has no text left to wait for.
scored(store, "One", {uid: 0.9})
assert enrich.plan(config(tmp_path), store) == []
@needs_waiting
def test_a_search_batch_counts_a_404_as_read_as_a_board_does(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
search(store)
one, two = search_role(store, "jobbank:cph", "1"), search_role(store, "jobbank:cph", "2")
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[one, two])
assert out["filled"] == 0 and out["read"] == 2
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == [
"jobbank:cph:1", "jobbank:cph:2"]
def test_a_reader_is_found_for_a_board_and_a_search_and_one_without_detail_is_refused(
web: dict[str, Any]) -> None:
assert ats.reader_of("fake:acme") is readers.HOSTS["fake"]
assert ats.reader_of("jobbank:cph") is readers.PROVIDERS["jobbank"]
assert ats.reader_of("nowhere:x") is None
with httpx.Client() as client:
for target in ("jobindex:cph", "plain:acme", "nowhere:x"):
with pytest.raises(ValueError, match="no reader"):
ats.read_detail(client, target, {"source_id": "1", "url": BASE})
with pytest.raises(ValueError, match="no reader"):
ats.read_board_details({}, target, [{"source_id": "1", "url": BASE}],
client=client)
# ------------------------------------------------------------------ cascade
class _Context:
+82
View File
@@ -8,12 +8,15 @@ from pathlib import Path
import pytest
import httpx
from app.jobs import posting
from app.storage import Store
from desk.models import Item
from desk.tools import scrape
from desk.tools.article import ArticleContent
from desk.tools.checks import Page
from plugins import ats
from prepped import upsert
ROOT = Path(__file__).resolve().parents[1]
@@ -164,3 +167,82 @@ def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
web = Web(monkeypatch, {}, {LINK: "The real posting."})
assert posting.fetch_posting(store, uid).article_text == "The real posting."
assert web.asked == [LINK]
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
AD_URL = "https://jobbank.dk/job/1234/acme/role"
def search_role(store: Store, provider: str = "jobbank", url: str = AD_URL) -> str:
"""A role of a saved search as a pull stores it: `provider:id`, source
`search`, the search's target in `raw`."""
target = f"{provider}:cph"
sub = store.add_subscription("search", target, label=target, params={"q": "x"})
row = {"source_id": "1234", "title": "Researcher", "url": url, "description": "",
"company": "Acme A/S"}
item = ats.item_of(row, target, target)
item.source = "search"
upsert(store, item, sub)
return item.uid
class Pages:
"""`ats._get` answering from `answers` (URL -> body or the exception to
raise), remembering what it was asked."""
def __init__(self, monkeypatch: pytest.MonkeyPatch, answers: dict[str, object]) -> None:
self.asked: list[str] = []
self.answers = answers
monkeypatch.setattr(ats, "_get", self.get)
def get(self, client: object, url: str, host: str, slug: str, **kw: object) -> bytes:
self.asked.append(url)
answer = self.answers[url]
if isinstance(answer, Exception):
raise answer
return answer # type: ignore[return-value]
def test_a_search_role_whose_reader_has_detail_is_read_by_its_detail(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = search_role(store)
pages = Pages(monkeypatch, {AD_URL: AD})
web = Web(monkeypatch, {}, {})
got = posting.fetch_posting(store, uid)
assert got.article_text and got.article_at and got.posting_url == ""
assert pages.asked == [AD_URL] and web.asked == [] # no scrape of the link
with pytest.raises(posting.AlreadyFetched):
posting.fetch_posting(store, uid)
@pytest.mark.parametrize(("answer", "message"), [
(httpx.HTTPStatusError("404", request=httpx.Request("GET", AD_URL),
response=httpx.Response(404, request=httpx.Request("GET", AD_URL))),
"gone"),
(httpx.HTTPStatusError("429", request=httpx.Request("GET", AD_URL),
response=httpx.Response(429, request=httpx.Request("GET", AD_URL))),
"refusing"),
(httpx.ConnectError("down"), "could not be reached"),
(b"<html>Just a moment...</html>", "Nothing readable"),
(b'<script type="application/ld+json">{"@type": "JobPosting"}</script>', "no readable text"),
])
def test_a_failed_detail_read_says_why(store: Store, monkeypatch: pytest.MonkeyPatch,
answer: object, message: str) -> None:
uid = search_role(store)
Pages(monkeypatch, {AD_URL: answer})
with pytest.raises(posting.Unreadable, match=message) as failure:
posting.fetch_posting(store, uid)
assert failure.value.status == 422
stored = store.role(uid)
assert stored is not None and stored.article_text == ""
def test_a_search_role_without_detail_is_scraped_from_its_link(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = search_role(store, "adzuna", "https://www.adzuna.dk/details/1")
pages = Pages(monkeypatch, {})
Web(monkeypatch, {}, {"https://www.adzuna.dk/details/1": "The posting."})
assert posting.fetch_posting(store, uid).article_text == "The posting."
assert pages.asked == []