Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
07e3bb4756
commit
b148647ac3
5 files changed
+346
-40
No files matched your search
@@ -239,6 +239,122 @@ def test_a_board_switched_off_since_the_plan_reads_nothing(
|
||||
assert web["asked"] == []
|
||||
|
||||
|
||||
# ----------------------------------------------------------- search roles
|
||||
|
||||
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
|
||||
AD_URL = "https://jobbank.dk/job/1234/acme/role"
|
||||
|
||||
|
||||
#: `Store.enrich_waiting` hands over `source = 'ats'` roles only, and a search's
|
||||
#: are `search`: until it takes both, the tests that need its answer fail.
|
||||
needs_waiting = pytest.mark.xfail(
|
||||
reason="Store.enrich_waiting reads source 'ats' only", strict=False)
|
||||
|
||||
|
||||
def search(store: Store, target: str = "jobbank:cph") -> Any:
|
||||
return store.add_subscription("search", target, label=target, params={"q": "x"})
|
||||
|
||||
|
||||
def search_role(store: Store, target: str, native: str, *, provider: str = "") -> str:
|
||||
"""A role of search `target` as a pull stores it: `provider:id`, the
|
||||
employer in `raw` and no text."""
|
||||
provider = provider or target.partition(":")[0]
|
||||
sub = next(s for s in store.subscriptions(source="search") if s.target == target)
|
||||
row = {"source_id": native, "title": "Researcher", "url": AD_URL.replace("1234", native),
|
||||
"description": "", "company": "Acme A/S", "places": [{"text": "Fyn", "remote": None}]}
|
||||
item = ats.item_of(row, target, sub.label)
|
||||
item.source = "search"
|
||||
assert item.source_id == f"{provider}:{native}"
|
||||
store.upsert_role(item, sub)
|
||||
return item.uid
|
||||
|
||||
|
||||
@needs_waiting
|
||||
def test_a_search_whose_reader_has_detail_is_planned_like_a_board(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
search(store, "jobindex:cph") # no `detail`: left out
|
||||
board(store, "fake:acme")
|
||||
a, b = search_role(store, "jobbank:cph", "1234"), search_role(store, "jobbank:cph", "99")
|
||||
j = search_role(store, "jobindex:cph", "7")
|
||||
f = role(store, "fake:acme", "1")
|
||||
scored(store, "One", {a: 0.9, b: 0.6, j: 0.95, f: 0.7})
|
||||
|
||||
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [a, b]), ("fake:acme", [f])]
|
||||
# The budgets are a board's: `per_board` and `per_run` count the search as one.
|
||||
assert enrich.plan(config(tmp_path, per_board=1), store) == [("jobbank:cph", [a]),
|
||||
("fake:acme", [f])]
|
||||
assert enrich.plan(config(tmp_path, per_run=2), store) == [("jobbank:cph", [a]),
|
||||
("fake:acme", [f])]
|
||||
# A failing search is left out, as a failing board is.
|
||||
store.state["search:failing:jobbank:cph"] = "{}"
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [f])]
|
||||
|
||||
|
||||
@needs_waiting
|
||||
def test_a_search_role_read_waits_for_the_retry_and_a_failed_one_goes_last(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
one, two, three = (search_role(store, "jobbank:cph", n) for n in ("1", "2", "3"))
|
||||
scored(store, "One", {one: 0.9, two: 0.8, three: 0.7})
|
||||
store.state["ats:detailed:jobbank:cph"] = json.dumps(["jobbank:cph:1", "jobbank:cph:2"])
|
||||
store.state["ats:detailfailed:jobbank:cph"] = json.dumps(["jobbank:cph:2"])
|
||||
store.state["ats:detailempty:jobbank:cph"] = json.dumps(
|
||||
{"jobbank:cph:1": datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"jobbank:cph:2": "2020-01-01T00:00:00+00:00"})
|
||||
|
||||
# 1 gave no text a moment ago and waits; 2 failed and is due, after 3.
|
||||
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [three, two])]
|
||||
|
||||
|
||||
@needs_waiting
|
||||
def test_a_batch_reads_a_search_roles_page_and_stores_its_text(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
uid = search_role(store, "jobbank:cph", "1234")
|
||||
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32))])
|
||||
web["answers"][AD_URL] = AD
|
||||
|
||||
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[uid])
|
||||
|
||||
assert out == {"target": "jobbank:cph", "read": 1, "filled": 1, "texts": 1}
|
||||
assert web["asked"] == [AD_URL]
|
||||
got = store.role(uid)
|
||||
assert got is not None and got.description
|
||||
assert (got.source, got.source_id, got.company) == ("search", "jobbank:1234", "Acme A/S")
|
||||
assert got.raw["search"] == "jobbank:cph" and got.raw["provider"] == "jobbank"
|
||||
assert store.vectors("fake") == {}
|
||||
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == ["jobbank:cph:1234"]
|
||||
# Read, it is not planned again, and has no text left to wait for.
|
||||
scored(store, "One", {uid: 0.9})
|
||||
assert enrich.plan(config(tmp_path), store) == []
|
||||
|
||||
|
||||
@needs_waiting
|
||||
def test_a_search_batch_counts_a_404_as_read_as_a_board_does(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
one, two = search_role(store, "jobbank:cph", "1"), search_role(store, "jobbank:cph", "2")
|
||||
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[one, two])
|
||||
assert out["filled"] == 0 and out["read"] == 2
|
||||
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == [
|
||||
"jobbank:cph:1", "jobbank:cph:2"]
|
||||
|
||||
|
||||
def test_a_reader_is_found_for_a_board_and_a_search_and_one_without_detail_is_refused(
|
||||
web: dict[str, Any]) -> None:
|
||||
assert ats.reader_of("fake:acme") is readers.HOSTS["fake"]
|
||||
assert ats.reader_of("jobbank:cph") is readers.PROVIDERS["jobbank"]
|
||||
assert ats.reader_of("nowhere:x") is None
|
||||
with httpx.Client() as client:
|
||||
for target in ("jobindex:cph", "plain:acme", "nowhere:x"):
|
||||
with pytest.raises(ValueError, match="no reader"):
|
||||
ats.read_detail(client, target, {"source_id": "1", "url": BASE})
|
||||
with pytest.raises(ValueError, match="no reader"):
|
||||
ats.read_board_details({}, target, [{"source_id": "1", "url": BASE}],
|
||||
client=client)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cascade
|
||||
|
||||
class _Context:
|
||||
|
||||
@@ -8,12 +8,15 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import httpx
|
||||
|
||||
from app.jobs import posting
|
||||
from app.storage import Store
|
||||
from desk.models import Item
|
||||
from desk.tools import scrape
|
||||
from desk.tools.article import ArticleContent
|
||||
from desk.tools.checks import Page
|
||||
from plugins import ats
|
||||
from prepped import upsert
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -164,3 +167,82 @@ def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
|
||||
web = Web(monkeypatch, {}, {LINK: "The real posting."})
|
||||
assert posting.fetch_posting(store, uid).article_text == "The real posting."
|
||||
assert web.asked == [LINK]
|
||||
|
||||
|
||||
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
|
||||
AD_URL = "https://jobbank.dk/job/1234/acme/role"
|
||||
|
||||
|
||||
def search_role(store: Store, provider: str = "jobbank", url: str = AD_URL) -> str:
|
||||
"""A role of a saved search as a pull stores it: `provider:id`, source
|
||||
`search`, the search's target in `raw`."""
|
||||
target = f"{provider}:cph"
|
||||
sub = store.add_subscription("search", target, label=target, params={"q": "x"})
|
||||
row = {"source_id": "1234", "title": "Researcher", "url": url, "description": "",
|
||||
"company": "Acme A/S"}
|
||||
item = ats.item_of(row, target, target)
|
||||
item.source = "search"
|
||||
upsert(store, item, sub)
|
||||
return item.uid
|
||||
|
||||
|
||||
class Pages:
|
||||
"""`ats._get` answering from `answers` (URL -> body or the exception to
|
||||
raise), remembering what it was asked."""
|
||||
|
||||
def __init__(self, monkeypatch: pytest.MonkeyPatch, answers: dict[str, object]) -> None:
|
||||
self.asked: list[str] = []
|
||||
self.answers = answers
|
||||
monkeypatch.setattr(ats, "_get", self.get)
|
||||
|
||||
def get(self, client: object, url: str, host: str, slug: str, **kw: object) -> bytes:
|
||||
self.asked.append(url)
|
||||
answer = self.answers[url]
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer # type: ignore[return-value]
|
||||
|
||||
|
||||
def test_a_search_role_whose_reader_has_detail_is_read_by_its_detail(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = search_role(store)
|
||||
pages = Pages(monkeypatch, {AD_URL: AD})
|
||||
web = Web(monkeypatch, {}, {})
|
||||
|
||||
got = posting.fetch_posting(store, uid)
|
||||
|
||||
assert got.article_text and got.article_at and got.posting_url == ""
|
||||
assert pages.asked == [AD_URL] and web.asked == [] # no scrape of the link
|
||||
with pytest.raises(posting.AlreadyFetched):
|
||||
posting.fetch_posting(store, uid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("answer", "message"), [
|
||||
(httpx.HTTPStatusError("404", request=httpx.Request("GET", AD_URL),
|
||||
response=httpx.Response(404, request=httpx.Request("GET", AD_URL))),
|
||||
"gone"),
|
||||
(httpx.HTTPStatusError("429", request=httpx.Request("GET", AD_URL),
|
||||
response=httpx.Response(429, request=httpx.Request("GET", AD_URL))),
|
||||
"refusing"),
|
||||
(httpx.ConnectError("down"), "could not be reached"),
|
||||
(b"<html>Just a moment...</html>", "Nothing readable"),
|
||||
(b'<script type="application/ld+json">{"@type": "JobPosting"}</script>', "no readable text"),
|
||||
])
|
||||
def test_a_failed_detail_read_says_why(store: Store, monkeypatch: pytest.MonkeyPatch,
|
||||
answer: object, message: str) -> None:
|
||||
uid = search_role(store)
|
||||
Pages(monkeypatch, {AD_URL: answer})
|
||||
with pytest.raises(posting.Unreadable, match=message) as failure:
|
||||
posting.fetch_posting(store, uid)
|
||||
assert failure.value.status == 422
|
||||
stored = store.role(uid)
|
||||
assert stored is not None and stored.article_text == ""
|
||||
|
||||
|
||||
def test_a_search_role_without_detail_is_scraped_from_its_link(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = search_role(store, "adzuna", "https://www.adzuna.dk/details/1")
|
||||
pages = Pages(monkeypatch, {})
|
||||
Web(monkeypatch, {}, {"https://www.adzuna.dk/details/1": "The posting."})
|
||||
assert posting.fetch_posting(store, uid).article_text == "The posting."
|
||||
assert pages.asked == []
|
||||
Reference in new issue
Block a user