"""`fetch_posting`: a role's posting text on demand, and for a Jobindex role the page its "Se jobbet" button leads to. No network: `scrape` is stubbed.""" from __future__ import annotations from datetime import UTC, datetime from pathlib import Path import pytest from app.jobs import posting from app.storage import Store from desk.models import Item from desk.tools import scrape from desk.tools.article import ArticleContent from desk.tools.checks import Page from prepped import upsert ROOT = Path(__file__).resolve().parents[1] VIS_JOB = "https://www.jobindex.dk/vis-job/h1702752" LINK = ("https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring" "?promotion=2220670-jobindex") PAGE = (ROOT / "tests" / "fixtures" / "jobindex" / "vis-job-employer.html").read_text("utf-8") @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") def role(store: Store, url: str = "https://boards.example/acme/1", source_id: str = "1", **raw: object) -> str: item = Item(source_id=source_id, location=url, title="Engineer", body="A teaser.", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw)) item.source = "ats" upsert(store, item) return item.uid def jobindex_role(store: Store) -> str: return role(store, VIS_JOB, provider="jobindex") def content(text: str, url: str = "") -> scrape.Scraped: return scrape.Scraped(ArticleContent(format="plain", content=text, source_url=url, method="test"), "", "http", ()) def failed(category: str) -> scrape.Scraped: return scrape.Scraped(None, category, "http", ()) class Web: """Stands in for the network: `pages` maps a URL to what `scrape.HTTP` answers with, `texts` a URL to what the article of that page reads as.""" def __init__(self, monkeypatch: pytest.MonkeyPatch, pages: dict[str, str], texts: dict[str, str]) -> None: self.asked: list[str] = [] self.pages, self.texts = pages, texts monkeypatch.setattr(scrape, "HTTP", scrape.Getter("http", self.fetch)) monkeypatch.setattr(scrape, "STANDARD", {"http": scrape.HTTP}) monkeypatch.setattr(scrape, "scrape", self.scrape) def fetch(self, url: str) -> Page | None: self.asked.append(url) html = self.pages.get(url) return None if html is None else Page(status=200, html=html, url=url, headers={}) def scrape(self, url: str, getters: object = None, first: str = "http") -> scrape.Scraped: if getters is not None: # the page is fetched by the getter it was handed page = getters["http"].fetch(url) # type: ignore[index] return content(self.texts[url], url) if page and url in self.texts else failed("empty") self.asked.append(url) return content(self.texts[url], url) if url in self.texts else failed("not_found") def test_a_role_that_cannot_be_fetched_says_why(store: Store, monkeypatch: pytest.MonkeyPatch) -> None: Web(monkeypatch, {}, {}) with pytest.raises(posting.UnknownRole) as unknown: posting.fetch_posting(store, "nope") assert (unknown.value.status, str(unknown.value)) == (404, "no such role: nope") bare = role(store, url="") with pytest.raises(posting.NoLink) as nolink: posting.fetch_posting(store, bare) assert nolink.value.status == 422 def test_an_ordinary_role_is_scraped_from_its_link_once(store: Store, monkeypatch: pytest.MonkeyPatch) -> None: uid = role(store) web = Web(monkeypatch, {}, {}) with pytest.raises(posting.Unreadable) as gone: posting.fetch_posting(store, uid) assert gone.value.status == 422 and "gone" in str(gone.value) assert web.asked == ["https://boards.example/acme/1"] web.texts["https://boards.example/acme/1"] = " The whole posting.\n" got = posting.fetch_posting(store, uid) assert got.article_text == "The whole posting." and got.article_at assert got.posting_url == "" with pytest.raises(posting.AlreadyFetched) as again: posting.fetch_posting(store, uid) assert again.value.status == 409 def test_an_unknown_failure_reads_as_nothing_readable(store: Store, monkeypatch: pytest.MonkeyPatch) -> None: uid = role(store) web = Web(monkeypatch, {}, {}) monkeypatch.setattr(web, "scrape", lambda url, *a, **k: failed("weird")) monkeypatch.setattr(scrape, "scrape", web.scrape) with pytest.raises(posting.Unreadable, match="Nothing readable"): posting.fetch_posting(store, uid) def test_a_jobindex_role_follows_its_se_jobbet_link_and_keeps_it( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: uid = jobindex_role(store) web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad.", LINK: "The real posting."}) got = posting.fetch_posting(store, uid) assert got.article_text == "The real posting." assert got.posting_url == LINK and got.original_url == VIS_JOB assert web.asked == [VIS_JOB, LINK] stored = store.role(uid) assert stored is not None and stored.posting_url == LINK def test_a_jobindex_role_falls_back_to_the_ads_own_text( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: # The link gives nothing readable: the link is kept, the vis-job text stored. uid = jobindex_role(store) web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad."}) got = posting.fetch_posting(store, uid) assert got.article_text == "The Jobindex ad." and got.posting_url == LINK assert web.asked == [VIS_JOB, LINK] # No button on the page: the ad's own text, no link, one request. other = role(store, "https://www.jobindex.dk/vis-job/r2", "2", provider="jobindex") web = Web(monkeypatch, {"https://www.jobindex.dk/vis-job/r2": "

Ingen knap

"}, {"https://www.jobindex.dk/vis-job/r2": "Ad two."}) got = posting.fetch_posting(store, other) assert got.article_text == "Ad two." and got.posting_url == "" assert web.asked == ["https://www.jobindex.dk/vis-job/r2"] def test_a_jobindex_role_whose_pages_read_as_nothing_is_unreadable_but_keeps_the_link( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: uid = jobindex_role(store) Web(monkeypatch, {VIS_JOB: PAGE}, {}) with pytest.raises(posting.Unreadable) as error: posting.fetch_posting(store, uid) assert error.value.failure == "not_found" stored = store.role(uid) assert stored is not None and stored.posting_url == LINK and stored.article_text == "" def test_a_kept_link_is_followed_without_fetching_the_ad_page_again( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: uid = jobindex_role(store) store.set_posting_url(uid, LINK) web = Web(monkeypatch, {}, {LINK: "The real posting."}) assert posting.fetch_posting(store, uid).article_text == "The real posting." assert web.asked == [LINK]