From b148647ac3d138ec6d3088592bd50e13c6b95b8d Mon Sep 17 00:00:00 2001 From: Wolf Date: Sat, 3 Oct 2026 12:29:19 +0200 Subject: [PATCH] Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail. Co-Authored-By: Claude Sonnet 5.5 --- app/jobs/enrich.py | 82 ++++++++++++++++++----------- app/jobs/posting.py | 53 ++++++++++++++++++- plugins/ats.py | 53 ++++++++++++++++--- tests/test_enrich.py | 116 ++++++++++++++++++++++++++++++++++++++++++ tests/test_posting.py | 82 +++++++++++++++++++++++++++++ 5 files changed, 346 insertions(+), 40 deletions(-) diff --git a/app/jobs/enrich.py b/app/jobs/enrich.py index f6f98e9..67ea99e 100644 --- a/app/jobs/enrich.py +++ b/app/jobs/enrich.py @@ -2,18 +2,22 @@ Some hosts list a role without its text and state it on the role's own page only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest -with a `detail`). The pull reads listings only; this stage reads those pages -after scoring, so the calls go where the ranking is: +with a `detail`, and a search provider such as Jobbank). The pull reads +listings only; this stage reads those pages after scoring, so the calls go +where the ranking is. A saved search whose reader has `detail` is treated as +a board under its target `provider:name`: the same bar, budgets and state +keys, and its failing record is `search:failing:`. 1. `enrich_waiting` hands over the open roles without text that an enabled agent with a prompt ranks ungated among its best `enrich.top` and at `enrich.min_score` or above, best first by the highest such score. A role below that bar for every agent gets no call. -2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most - `enrich.per_board` from one board and `enrich.per_run` in all, leaving - out the roles already read, unless their read failed or gave no text - `enrich.retry_days` ago (`plugins.ats.detail_due`). -3. One batch per board, on the `http` pool, reads them +2. They are taken in that order, those whose read failed last, on boards and + searches whose reader has `detail`, at most `enrich.per_board` from one + board and `enrich.per_run` in all, leaving out the roles already read, + unless their read failed or gave no text `enrich.retry_days` ago + (`plugins.ats.detail_due`). +3. One batch per board or search, on the `http` pool, reads them (`plugins.ats.read_board_details`, which keeps `ats:detailed:`, `ats:detailfailed:` and `ats:detailempty:`) and stores what each page filled. `Store.upsert_role` drops the vector of a role @@ -37,6 +41,7 @@ import httpx from app.config import Config from app.jobs import stages +from app.models import Subscription from app.storage import Store from pipeline import NULL, Batch, RunContext, Tally, Task, job @@ -53,7 +58,14 @@ def _ats() -> Any: return ats -def _row(found: dict[str, Any], target: str) -> dict[str, Any]: +def _native(found: dict[str, Any], sub: Subscription) -> str: + """The host's own id of a stored role: a board's roles are stored as + `host:slug:id`, a search's as `provider:id`.""" + prefix = sub.target if sub.source == "ats" else sub.target.partition(":")[0] + return str(found["source_id"]).removeprefix(f"{prefix}:") + + +def _row(found: dict[str, Any], sub: Subscription) -> dict[str, Any]: """A stored role back as the row its reader listed: the stored `raw` with the host's own id, the title, the link and the date.""" try: @@ -65,29 +77,36 @@ def _row(found: dict[str, Any], target: str) -> dict[str, Any]: posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or "")) except ValueError: posted = None - return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"), + return raw | {"source_id": _native(found, sub), "title": found["title"], "url": found["url"], "description": "", "posted": posted} +def _readable(store: Store) -> dict[int, Subscription]: + """The fetchable boards and searches whose reader has `detail`, by + subscription id.""" + ats = _ats() + return {sub.id: sub for source in ("ats", "search") + for sub in store.subscriptions(source=source, fetchable=True) + if hasattr(ats.reader_of(sub.target), "detail")} + + def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]: - """`(target, uids)` per board, best board first, within the budgets: the - roles `enrich_waiting` ranks, on fetchable boards whose reader has - `detail` and that are not failing, and due for a read.""" + """`(target, uids)` per board or search, best first, within the budgets: + the roles `enrich_waiting` ranks, on fetchable boards and searches whose + reader has `detail` and that are not failing, and due for a read.""" settings = config.enrich ats = _ats() - boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)} + subs = _readable(store) by_board: dict[str, list[dict[str, Any]]] = defaultdict(list) for found in store.enrich_waiting(settings.top, settings.min_score): - target = boards.get(found["subscription_id"]) - if target is None: + sub = subs.get(found["subscription_id"]) + if sub is None or f"{sub.source}:failing:{sub.target}" in store.state: continue - module = ats.readers.HOSTS.get(target.partition(":")[0]) - if module is None or not hasattr(module, "detail") \ - or f"ats:failing:{target}" in store.state: - continue - by_board[target].append(found) - due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found], + # The id the board's per-role state keeps: `host:slug:id`. + found["compound"] = f"{sub.target}:{_native(found, sub)}" + by_board[sub.target].append(found) + due = {target: set(ats.detail_due(store.state, target, [f["compound"] for f in found], retry_days=settings.retry_days)) for target, found in by_board.items()} # Best first; roles whose read failed after every other, so a page that @@ -95,13 +114,13 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]: failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1]) for target in by_board} ranked = sorted((f for found in by_board.values() for f in found), - key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]], + key=lambda f: (f["compound"] in failed[subs[f["subscription_id"]].target], -float(f["score"] or 0), f["uid"])) picked: dict[str, list[str]] = {} total = 0 for found in ranked: - target = boards[found["subscription_id"]] - if found["source_id"] not in due[target]: + target = subs[found["subscription_id"]].target + if found["compound"] not in due[target]: continue if total >= settings.per_run: break @@ -114,7 +133,8 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]: def batches(config: Config, store: Store) -> list[Batch]: - """The finder: one batch per board, nothing while the stage is off.""" + """The finder: one batch per board or search, nothing while the stage is + off.""" if not stages.stage_on(config, store, "enrich"): return [] return [Batch({"target": target, "uids": uids}, @@ -125,16 +145,16 @@ def batches(config: Config, store: Store) -> list[Batch]: @job(KIND, "Reading roles on their own pages", resource="http") def run_batch(config: Config, store: Store, *, target: str, uids: list[str], context: RunContext = NULL) -> dict[str, Any]: - """Read one board's roles in `uids` on their own pages, in that order, - and store what each page filled. `filled` is how many roles changed, - `texts` how many of them now have text.""" + """Read one board's or search's roles in `uids` on their own pages, in + that order, and store what each page filled. `filled` is how many roles + changed, `texts` how many of them now have text.""" ats = _ats() - subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target] + subs = [s for s in _readable(store).values() if s.target == target] if not subs or not uids: return {"target": target, "filled": 0, "texts": 0} sub = subs[0] found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)} - rows = [_row(found[uid], target) for uid in uids if uid in found] + rows = [_row(found[uid], sub) for uid in uids if uid in found] context.check() with httpx.Client() as client: filled = ats.read_board_details(store.state, target, rows, client=client, @@ -143,7 +163,7 @@ def run_batch(config: Config, store: Store, *, target: str, uids: list[str], texts = 0 for row in filled: item = ats.item_of(row, target, label) - item.source = "ats" + item.source = sub.source store.upsert_role(item, sub) texts += bool(row.get("description")) log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled), diff --git a/app/jobs/posting.py b/app/jobs/posting.py index 9d5fc2a..f74ca1c 100644 --- a/app/jobs/posting.py +++ b/app/jobs/posting.py @@ -5,6 +5,11 @@ link, keep the text as `article_text`. It is on demand, never in the pull, because a fetch of a Jobindex page counts as a click in Jobindex's statistics and one click for a posting somebody opened is fair. +A role of a search whose reader has `detail` (Jobbank) is read through that +`detail`, which reads the ad's own page and states its text, rather than +scraping the link; a failure of that read is an `Unreadable` with the +category its HTTP status or exception names. + A Jobindex role links to Jobindex's own `vis-job` page, which carries a short ad and a "Se jobbet" button to the posting itself: the employer's site, The Hub or Jobindex's `jobannonce`. That button's link is found, kept as @@ -19,9 +24,14 @@ message the route answers with. from __future__ import annotations +from datetime import datetime +from typing import Any + +import httpx + from app.models import Role from app.storage import Store -from desk.tools import scrape +from desk.tools import failures, scrape from desk.tools.checks import Page from plugins.readers import jobindex @@ -113,6 +123,43 @@ def _jobindex_text(store: Store, role: Role) -> str: raise Unreadable(failure or own.failure) +def _detail_target(role: Role) -> str: + """The search target of a role whose reader has `detail`, else ''.""" + from plugins import ats + + target = str(role.raw.get("search") or "") + if role.source == "search" and hasattr(ats.reader_of(target), "detail"): + return target + return "" + + +def _detail_text(role: Role, target: str) -> str: + """The text of a search role, read by its reader's `detail` on the ad's + own page as the pull's enrich stage reads it. Raises `Unreadable` when + the read fails or the page states no text.""" + from plugins import ats + + provider = target.partition(":")[0] + try: + posted: datetime | None = datetime.fromisoformat(role.published_at) + except ValueError: + posted = None + row: dict[str, Any] = role.raw | { + "source_id": role.source_id.removeprefix(f"{provider}:"), "title": role.title, + "url": role.original_url, "description": "", "posted": posted} + try: + with httpx.Client() as client: + found = ats.read_detail(client, target, row) + except httpx.HTTPStatusError as exc: + raise Unreadable(failures.of_status(exc.response.status_code)) from exc + except Exception as exc: + category = failures.of_exception(exc) + raise Unreadable("" if category == failures.UNKNOWN else category) from exc + if not (text := str(found.get("description") or "").strip()): + raise Unreadable("empty") + return text + + def fetch_posting(store: Store, uid: str) -> Role: """Fetch the posting page once and keep it as plain text, for a role whose description is a teaser. Synchronous: one request the person asked @@ -124,7 +171,9 @@ def fetch_posting(store: Store, uid: str) -> Role: raise AlreadyFetched("The posting text is already fetched.") if not role.original_url: raise NoLink("This role has no link to fetch the posting from.") - if _is_jobindex(role): + if target := _detail_target(role): + text = _detail_text(role, target) + elif _is_jobindex(role): text = _jobindex_text(store, role) else: got = scrape.scrape(role.original_url) diff --git a/plugins/ats.py b/plugins/ats.py index 08308a6..c40d3a5 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -38,7 +38,9 @@ that read once it is complete, rather than by a second request. A pull reads listings only. A host that states a role's text on the role's own page (`text="detail"`) is read there by `app/jobs/enrich.py` after -scoring, best roles first, through `read_board_details`. +scoring, best roles first, through `read_board_details`. A search provider +whose reader has `detail` (Jobbank) is read the same way, under its search +target `provider:name`; `read_detail` is the one read the Fetch button makes. The limits below are read when a board is read, so a test that replaces one on this module reaches the runner. @@ -53,6 +55,7 @@ import sys from collections.abc import Callable, Iterator from datetime import UTC, datetime, timedelta from pathlib import Path +from types import ModuleType from typing import Any import httpx @@ -114,11 +117,18 @@ def _limits(stale_days: int | None = None) -> _runner.Limits: stale_days=stale_days) +def reader_of(target: str) -> ModuleType | None: + """The reader of a board (`host:slug`) or of a search (`provider:name`), + None for a name that is neither. The two never share a name.""" + host = target.partition(":")[0] + return readers.HOSTS.get(host) or readers.PROVIDERS.get(host) + + def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: - """The hosts a request or a redirect may land on for one board: what its - host module's `allowed` says. Anything else is refused, at the first hop - and at every redirect.""" - module = readers.HOSTS.get(host) + """The hosts a request or a redirect may land on for one board or search: + what its module's `allowed` says. Anything else is refused, at the first + hop and at every redirect.""" + module = reader_of(host) return module.allowed(slug) if module is not None else lambda h: False @@ -540,7 +550,9 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http are written and changed by what this read did, so a pull that dropped ids meanwhile keeps its drop.""" host, _, slug = target.partition(":") - module = readers.HOSTS[host] + module = reader_of(target) + if module is None or not hasattr(module, "detail"): + raise ValueError(f"{target!r} has no reader that reads a role on its own page") keys = detail_keys(target) now = now or datetime.now(UTC) @@ -582,12 +594,39 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http return filled +def read_detail(client: httpx.Client, target: str, row: Row) -> dict[str, Any]: + """One role of board or search `target` read on its own page: what the + reader's `detail` returns, for the Fetch button. No state is kept and no + field is filtered. Raises what `detail` raises, and ValueError for a + target whose reader has none.""" + host, _, slug = target.partition(":") + module = reader_of(target) + if module is None or not hasattr(module, "detail"): + raise ValueError(f"{target!r} has no reader that reads a role on its own page") + return module.detail(_getter(client, host, slug), slug, row) + + def item_of(row: Row, target: str, label: str) -> Item: """One row of board `target` as the `Item` the store takes: its id compound, `host:slug:id`, as `roles.source_id` has it. A row without a - date is dated now, as `Item` itself would date it.""" + date is dated now, as `Item` itself would date it. A search's row is + stored as `plugins/search.py` stores it: `provider:id`, the employer from + `company` and `label` the search's own.""" host, _, slug = target.partition(":") description = row.get("description") or "" + if host not in readers.HOSTS: + return Item( + source_id=f"{host}:{row['source_id']}", + location=row["url"], + title=row["title"], + excerpt=description[:300], + body=description, + published_at=row.get("posted") or datetime.now(UTC), + subscription=label, + kind="role", + raw=_runner.raw_of(row) | {"company": row.get("company") or "", "provider": host, + "search": target}, + ) return Item( source_id=f"{host}:{slug}:{row['source_id']}", location=row["url"], diff --git a/tests/test_enrich.py b/tests/test_enrich.py index ed2190c..4dd2f8a 100644 --- a/tests/test_enrich.py +++ b/tests/test_enrich.py @@ -239,6 +239,122 @@ def test_a_board_switched_off_since_the_plan_reads_nothing( assert web["asked"] == [] +# ----------------------------------------------------------- search roles + +AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes() +AD_URL = "https://jobbank.dk/job/1234/acme/role" + + +#: `Store.enrich_waiting` hands over `source = 'ats'` roles only, and a search's +#: are `search`: until it takes both, the tests that need its answer fail. +needs_waiting = pytest.mark.xfail( + reason="Store.enrich_waiting reads source 'ats' only", strict=False) + + +def search(store: Store, target: str = "jobbank:cph") -> Any: + return store.add_subscription("search", target, label=target, params={"q": "x"}) + + +def search_role(store: Store, target: str, native: str, *, provider: str = "") -> str: + """A role of search `target` as a pull stores it: `provider:id`, the + employer in `raw` and no text.""" + provider = provider or target.partition(":")[0] + sub = next(s for s in store.subscriptions(source="search") if s.target == target) + row = {"source_id": native, "title": "Researcher", "url": AD_URL.replace("1234", native), + "description": "", "company": "Acme A/S", "places": [{"text": "Fyn", "remote": None}]} + item = ats.item_of(row, target, sub.label) + item.source = "search" + assert item.source_id == f"{provider}:{native}" + store.upsert_role(item, sub) + return item.uid + + +@needs_waiting +def test_a_search_whose_reader_has_detail_is_planned_like_a_board( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + search(store) + search(store, "jobindex:cph") # no `detail`: left out + board(store, "fake:acme") + a, b = search_role(store, "jobbank:cph", "1234"), search_role(store, "jobbank:cph", "99") + j = search_role(store, "jobindex:cph", "7") + f = role(store, "fake:acme", "1") + scored(store, "One", {a: 0.9, b: 0.6, j: 0.95, f: 0.7}) + + assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [a, b]), ("fake:acme", [f])] + # The budgets are a board's: `per_board` and `per_run` count the search as one. + assert enrich.plan(config(tmp_path, per_board=1), store) == [("jobbank:cph", [a]), + ("fake:acme", [f])] + assert enrich.plan(config(tmp_path, per_run=2), store) == [("jobbank:cph", [a]), + ("fake:acme", [f])] + # A failing search is left out, as a failing board is. + store.state["search:failing:jobbank:cph"] = "{}" + assert enrich.plan(config(tmp_path), store) == [("fake:acme", [f])] + + +@needs_waiting +def test_a_search_role_read_waits_for_the_retry_and_a_failed_one_goes_last( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + search(store) + one, two, three = (search_role(store, "jobbank:cph", n) for n in ("1", "2", "3")) + scored(store, "One", {one: 0.9, two: 0.8, three: 0.7}) + store.state["ats:detailed:jobbank:cph"] = json.dumps(["jobbank:cph:1", "jobbank:cph:2"]) + store.state["ats:detailfailed:jobbank:cph"] = json.dumps(["jobbank:cph:2"]) + store.state["ats:detailempty:jobbank:cph"] = json.dumps( + {"jobbank:cph:1": datetime.now(UTC).isoformat(timespec="seconds"), + "jobbank:cph:2": "2020-01-01T00:00:00+00:00"}) + + # 1 gave no text a moment ago and waits; 2 failed and is due, after 3. + assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [three, two])] + + +@needs_waiting +def test_a_batch_reads_a_search_roles_page_and_stores_its_text( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + search(store) + uid = search_role(store, "jobbank:cph", "1234") + store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32))]) + web["answers"][AD_URL] = AD + + out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[uid]) + + assert out == {"target": "jobbank:cph", "read": 1, "filled": 1, "texts": 1} + assert web["asked"] == [AD_URL] + got = store.role(uid) + assert got is not None and got.description + assert (got.source, got.source_id, got.company) == ("search", "jobbank:1234", "Acme A/S") + assert got.raw["search"] == "jobbank:cph" and got.raw["provider"] == "jobbank" + assert store.vectors("fake") == {} + assert json.loads(store.state["ats:detailed:jobbank:cph"]) == ["jobbank:cph:1234"] + # Read, it is not planned again, and has no text left to wait for. + scored(store, "One", {uid: 0.9}) + assert enrich.plan(config(tmp_path), store) == [] + + +@needs_waiting +def test_a_search_batch_counts_a_404_as_read_as_a_board_does( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + search(store) + one, two = search_role(store, "jobbank:cph", "1"), search_role(store, "jobbank:cph", "2") + out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[one, two]) + assert out["filled"] == 0 and out["read"] == 2 + assert json.loads(store.state["ats:detailed:jobbank:cph"]) == [ + "jobbank:cph:1", "jobbank:cph:2"] + + +def test_a_reader_is_found_for_a_board_and_a_search_and_one_without_detail_is_refused( + web: dict[str, Any]) -> None: + assert ats.reader_of("fake:acme") is readers.HOSTS["fake"] + assert ats.reader_of("jobbank:cph") is readers.PROVIDERS["jobbank"] + assert ats.reader_of("nowhere:x") is None + with httpx.Client() as client: + for target in ("jobindex:cph", "plain:acme", "nowhere:x"): + with pytest.raises(ValueError, match="no reader"): + ats.read_detail(client, target, {"source_id": "1", "url": BASE}) + with pytest.raises(ValueError, match="no reader"): + ats.read_board_details({}, target, [{"source_id": "1", "url": BASE}], + client=client) + + # ------------------------------------------------------------------ cascade class _Context: diff --git a/tests/test_posting.py b/tests/test_posting.py index f4ffcc6..81bb606 100644 --- a/tests/test_posting.py +++ b/tests/test_posting.py @@ -8,12 +8,15 @@ from pathlib import Path import pytest +import httpx + from app.jobs import posting from app.storage import Store from desk.models import Item from desk.tools import scrape from desk.tools.article import ArticleContent from desk.tools.checks import Page +from plugins import ats from prepped import upsert ROOT = Path(__file__).resolve().parents[1] @@ -164,3 +167,82 @@ def test_a_kept_link_is_followed_without_fetching_the_ad_page_again( web = Web(monkeypatch, {}, {LINK: "The real posting."}) assert posting.fetch_posting(store, uid).article_text == "The real posting." assert web.asked == [LINK] + + +AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes() +AD_URL = "https://jobbank.dk/job/1234/acme/role" + + +def search_role(store: Store, provider: str = "jobbank", url: str = AD_URL) -> str: + """A role of a saved search as a pull stores it: `provider:id`, source + `search`, the search's target in `raw`.""" + target = f"{provider}:cph" + sub = store.add_subscription("search", target, label=target, params={"q": "x"}) + row = {"source_id": "1234", "title": "Researcher", "url": url, "description": "", + "company": "Acme A/S"} + item = ats.item_of(row, target, target) + item.source = "search" + upsert(store, item, sub) + return item.uid + + +class Pages: + """`ats._get` answering from `answers` (URL -> body or the exception to + raise), remembering what it was asked.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, answers: dict[str, object]) -> None: + self.asked: list[str] = [] + self.answers = answers + monkeypatch.setattr(ats, "_get", self.get) + + def get(self, client: object, url: str, host: str, slug: str, **kw: object) -> bytes: + self.asked.append(url) + answer = self.answers[url] + if isinstance(answer, Exception): + raise answer + return answer # type: ignore[return-value] + + +def test_a_search_role_whose_reader_has_detail_is_read_by_its_detail( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + uid = search_role(store) + pages = Pages(monkeypatch, {AD_URL: AD}) + web = Web(monkeypatch, {}, {}) + + got = posting.fetch_posting(store, uid) + + assert got.article_text and got.article_at and got.posting_url == "" + assert pages.asked == [AD_URL] and web.asked == [] # no scrape of the link + with pytest.raises(posting.AlreadyFetched): + posting.fetch_posting(store, uid) + + +@pytest.mark.parametrize(("answer", "message"), [ + (httpx.HTTPStatusError("404", request=httpx.Request("GET", AD_URL), + response=httpx.Response(404, request=httpx.Request("GET", AD_URL))), + "gone"), + (httpx.HTTPStatusError("429", request=httpx.Request("GET", AD_URL), + response=httpx.Response(429, request=httpx.Request("GET", AD_URL))), + "refusing"), + (httpx.ConnectError("down"), "could not be reached"), + (b"Just a moment...", "Nothing readable"), + (b'', "no readable text"), +]) +def test_a_failed_detail_read_says_why(store: Store, monkeypatch: pytest.MonkeyPatch, + answer: object, message: str) -> None: + uid = search_role(store) + Pages(monkeypatch, {AD_URL: answer}) + with pytest.raises(posting.Unreadable, match=message) as failure: + posting.fetch_posting(store, uid) + assert failure.value.status == 422 + stored = store.role(uid) + assert stored is not None and stored.article_text == "" + + +def test_a_search_role_without_detail_is_scraped_from_its_link( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + uid = search_role(store, "adzuna", "https://www.adzuna.dk/details/1") + pages = Pages(monkeypatch, {}) + Web(monkeypatch, {}, {"https://www.adzuna.dk/details/1": "The posting."}) + assert posting.fetch_posting(store, uid).article_text == "The posting." + assert pages.asked == []