Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:29:19 +02:00
1 parent 07e3bb4756
commit b148647ac3
5 files changed
+346 -40

No files matched your search

+82
View File
@@ -8,12 +8,15 @@ from pathlib import Path
import pytest
import httpx
from app.jobs import posting
from app.storage import Store
from desk.models import Item
from desk.tools import scrape
from desk.tools.article import ArticleContent
from desk.tools.checks import Page
from plugins import ats
from prepped import upsert
ROOT = Path(__file__).resolve().parents[1]
@@ -164,3 +167,82 @@ def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
web = Web(monkeypatch, {}, {LINK: "The real posting."})
assert posting.fetch_posting(store, uid).article_text == "The real posting."
assert web.asked == [LINK]
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
AD_URL = "https://jobbank.dk/job/1234/acme/role"
def search_role(store: Store, provider: str = "jobbank", url: str = AD_URL) -> str:
"""A role of a saved search as a pull stores it: `provider:id`, source
`search`, the search's target in `raw`."""
target = f"{provider}:cph"
sub = store.add_subscription("search", target, label=target, params={"q": "x"})
row = {"source_id": "1234", "title": "Researcher", "url": url, "description": "",
"company": "Acme A/S"}
item = ats.item_of(row, target, target)
item.source = "search"
upsert(store, item, sub)
return item.uid
class Pages:
"""`ats._get` answering from `answers` (URL -> body or the exception to
raise), remembering what it was asked."""
def __init__(self, monkeypatch: pytest.MonkeyPatch, answers: dict[str, object]) -> None:
self.asked: list[str] = []
self.answers = answers
monkeypatch.setattr(ats, "_get", self.get)
def get(self, client: object, url: str, host: str, slug: str, **kw: object) -> bytes:
self.asked.append(url)
answer = self.answers[url]
if isinstance(answer, Exception):
raise answer
return answer # type: ignore[return-value]
def test_a_search_role_whose_reader_has_detail_is_read_by_its_detail(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = search_role(store)
pages = Pages(monkeypatch, {AD_URL: AD})
web = Web(monkeypatch, {}, {})
got = posting.fetch_posting(store, uid)
assert got.article_text and got.article_at and got.posting_url == ""
assert pages.asked == [AD_URL] and web.asked == [] # no scrape of the link
with pytest.raises(posting.AlreadyFetched):
posting.fetch_posting(store, uid)
@pytest.mark.parametrize(("answer", "message"), [
(httpx.HTTPStatusError("404", request=httpx.Request("GET", AD_URL),
response=httpx.Response(404, request=httpx.Request("GET", AD_URL))),
"gone"),
(httpx.HTTPStatusError("429", request=httpx.Request("GET", AD_URL),
response=httpx.Response(429, request=httpx.Request("GET", AD_URL))),
"refusing"),
(httpx.ConnectError("down"), "could not be reached"),
(b"<html>Just a moment...</html>", "Nothing readable"),
(b'<script type="application/ld+json">{"@type": "JobPosting"}</script>', "no readable text"),
])
def test_a_failed_detail_read_says_why(store: Store, monkeypatch: pytest.MonkeyPatch,
answer: object, message: str) -> None:
uid = search_role(store)
Pages(monkeypatch, {AD_URL: answer})
with pytest.raises(posting.Unreadable, match=message) as failure:
posting.fetch_posting(store, uid)
assert failure.value.status == 422
stored = store.role(uid)
assert stored is not None and stored.article_text == ""
def test_a_search_role_without_detail_is_scraped_from_its_link(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = search_role(store, "adzuna", "https://www.adzuna.dk/details/1")
pages = Pages(monkeypatch, {})
Web(monkeypatch, {}, {"https://www.adzuna.dk/details/1": "The posting."})
assert posting.fetch_posting(store, uid).article_text == "The posting."
assert pages.asked == []