Files
jobboard/tests/test_posting.py
T

249 lines
10 KiB
Python

"""`fetch_posting`: a role's posting text on demand, and for a Jobindex role
the page its "Se jobbet" button leads to. No network: `scrape` is stubbed."""
from __future__ import annotations
from datetime import UTC, datetime
from pathlib import Path
import pytest
import httpx
from app.jobs import posting
from app.storage import Store
from desk.models import Item
from desk.tools import scrape
from desk.tools.article import ArticleContent
from desk.tools.checks import Page
from plugins import ats
from prepped import upsert
ROOT = Path(__file__).resolve().parents[1]
VIS_JOB = "https://www.jobindex.dk/vis-job/h1702752"
LINK = ("https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring"
"?promotion=2220670-jobindex")
PAGE = (ROOT / "tests" / "fixtures" / "jobindex" / "vis-job-employer.html").read_text("utf-8")
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def role(store: Store, url: str = "https://boards.example/acme/1", source_id: str = "1",
**raw: object) -> str:
item = Item(source_id=source_id, location=url, title="Engineer", body="A teaser.",
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
item.source = "ats"
upsert(store, item)
return item.uid
def jobindex_role(store: Store) -> str:
return role(store, VIS_JOB, provider="jobindex")
def content(text: str, url: str = "") -> scrape.Scraped:
return scrape.Scraped(ArticleContent(format="plain", content=text, source_url=url,
method="test"), "", "http", ())
def failed(category: str) -> scrape.Scraped:
return scrape.Scraped(None, category, "http", ())
class Web:
"""Stands in for the network: `pages` maps a URL to what `scrape.HTTP`
answers with, `texts` a URL to what the article of that page reads as."""
def __init__(self, monkeypatch: pytest.MonkeyPatch, pages: dict[str, str],
texts: dict[str, str]) -> None:
self.asked: list[str] = []
self.pages, self.texts = pages, texts
monkeypatch.setattr(scrape, "HTTP", scrape.Getter("http", self.fetch))
monkeypatch.setattr(scrape, "STANDARD", {"http": scrape.HTTP})
monkeypatch.setattr(scrape, "scrape", self.scrape)
def fetch(self, url: str) -> Page | None:
self.asked.append(url)
html = self.pages.get(url)
return None if html is None else Page(status=200, html=html, url=url, headers={})
def scrape(self, url: str, getters: object = None, first: str = "http") -> scrape.Scraped:
if getters is not None: # the page is fetched by the getter it was handed
page = getters["http"].fetch(url) # type: ignore[index]
return content(self.texts[url], url) if page and url in self.texts else failed("empty")
self.asked.append(url)
return content(self.texts[url], url) if url in self.texts else failed("not_found")
def test_a_role_that_cannot_be_fetched_says_why(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
Web(monkeypatch, {}, {})
with pytest.raises(posting.UnknownRole) as unknown:
posting.fetch_posting(store, "nope")
assert (unknown.value.status, str(unknown.value)) == (404, "no such role: nope")
bare = role(store, url="")
with pytest.raises(posting.NoLink) as nolink:
posting.fetch_posting(store, bare)
assert nolink.value.status == 422
def test_an_ordinary_role_is_scraped_from_its_link_once(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
uid = role(store)
web = Web(monkeypatch, {}, {})
with pytest.raises(posting.Unreadable) as gone:
posting.fetch_posting(store, uid)
assert gone.value.status == 422 and "gone" in str(gone.value)
assert web.asked == ["https://boards.example/acme/1"]
web.texts["https://boards.example/acme/1"] = " The whole posting.\n"
got = posting.fetch_posting(store, uid)
assert got.article_text == "The whole posting." and got.article_at
assert got.posting_url == ""
with pytest.raises(posting.AlreadyFetched) as again:
posting.fetch_posting(store, uid)
assert again.value.status == 409
def test_an_unknown_failure_reads_as_nothing_readable(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
uid = role(store)
web = Web(monkeypatch, {}, {})
monkeypatch.setattr(web, "scrape", lambda url, *a, **k: failed("weird"))
monkeypatch.setattr(scrape, "scrape", web.scrape)
with pytest.raises(posting.Unreadable, match="Nothing readable"):
posting.fetch_posting(store, uid)
def test_a_jobindex_role_follows_its_se_jobbet_link_and_keeps_it(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad.", LINK: "The real posting."})
got = posting.fetch_posting(store, uid)
assert got.article_text == "The real posting."
assert got.posting_url == LINK and got.original_url == VIS_JOB
assert web.asked == [VIS_JOB, LINK]
stored = store.role(uid)
assert stored is not None and stored.posting_url == LINK
def test_a_jobindex_role_falls_back_to_the_ads_own_text(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
# The link gives nothing readable: the link is kept, the vis-job text stored.
uid = jobindex_role(store)
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad."})
got = posting.fetch_posting(store, uid)
assert got.article_text == "The Jobindex ad." and got.posting_url == LINK
assert web.asked == [VIS_JOB, LINK]
# No button on the page: the ad's own text, no link, one request.
other = role(store, "https://www.jobindex.dk/vis-job/r2", "2", provider="jobindex")
web = Web(monkeypatch, {"https://www.jobindex.dk/vis-job/r2": "<p>Ingen knap</p>"},
{"https://www.jobindex.dk/vis-job/r2": "Ad two."})
got = posting.fetch_posting(store, other)
assert got.article_text == "Ad two." and got.posting_url == ""
assert web.asked == ["https://www.jobindex.dk/vis-job/r2"]
def test_a_jobindex_role_whose_pages_read_as_nothing_is_unreadable_but_keeps_the_link(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
Web(monkeypatch, {VIS_JOB: PAGE}, {})
with pytest.raises(posting.Unreadable) as error:
posting.fetch_posting(store, uid)
assert error.value.failure == "not_found"
stored = store.role(uid)
assert stored is not None and stored.posting_url == LINK and stored.article_text == ""
def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
store.set_posting_url(uid, LINK)
web = Web(monkeypatch, {}, {LINK: "The real posting."})
assert posting.fetch_posting(store, uid).article_text == "The real posting."
assert web.asked == [LINK]
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
AD_URL = "https://jobbank.dk/job/1234/acme/role"
def search_role(store: Store, provider: str = "jobbank", url: str = AD_URL) -> str:
"""A role of a saved search as a pull stores it: `provider:id`, source
`search`, the search's target in `raw`."""
target = f"{provider}:cph"
sub = store.add_subscription("search", target, label=target, params={"q": "x"})
row = {"source_id": "1234", "title": "Researcher", "url": url, "description": "",
"company": "Acme A/S"}
item = ats.item_of(row, target, target)
item.source = "search"
upsert(store, item, sub)
return item.uid
class Pages:
"""`ats._get` answering from `answers` (URL -> body or the exception to
raise), remembering what it was asked."""
def __init__(self, monkeypatch: pytest.MonkeyPatch, answers: dict[str, object]) -> None:
self.asked: list[str] = []
self.answers = answers
monkeypatch.setattr(ats, "_get", self.get)
def get(self, client: object, url: str, host: str, slug: str, **kw: object) -> bytes:
self.asked.append(url)
answer = self.answers[url]
if isinstance(answer, Exception):
raise answer
return answer # type: ignore[return-value]
def test_a_search_role_whose_reader_has_detail_is_read_by_its_detail(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = search_role(store)
pages = Pages(monkeypatch, {AD_URL: AD})
web = Web(monkeypatch, {}, {})
got = posting.fetch_posting(store, uid)
assert got.article_text and got.article_at and got.posting_url == ""
assert pages.asked == [AD_URL] and web.asked == [] # no scrape of the link
with pytest.raises(posting.AlreadyFetched):
posting.fetch_posting(store, uid)
@pytest.mark.parametrize(("answer", "message"), [
(httpx.HTTPStatusError("404", request=httpx.Request("GET", AD_URL),
response=httpx.Response(404, request=httpx.Request("GET", AD_URL))),
"gone"),
(httpx.HTTPStatusError("429", request=httpx.Request("GET", AD_URL),
response=httpx.Response(429, request=httpx.Request("GET", AD_URL))),
"refusing"),
(httpx.ConnectError("down"), "could not be reached"),
(b"<html>Just a moment...</html>", "Nothing readable"),
(b'<script type="application/ld+json">{"@type": "JobPosting"}</script>', "no readable text"),
])
def test_a_failed_detail_read_says_why(store: Store, monkeypatch: pytest.MonkeyPatch,
answer: object, message: str) -> None:
uid = search_role(store)
Pages(monkeypatch, {AD_URL: answer})
with pytest.raises(posting.Unreadable, match=message) as failure:
posting.fetch_posting(store, uid)
assert failure.value.status == 422
stored = store.role(uid)
assert stored is not None and stored.article_text == ""
def test_a_search_role_without_detail_is_scraped_from_its_link(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = search_role(store, "adzuna", "https://www.adzuna.dk/details/1")
pages = Pages(monkeypatch, {})
Web(monkeypatch, {}, {"https://www.adzuna.dk/details/1": "The posting."})
assert posting.fetch_posting(store, uid).article_text == "The posting."
assert pages.asked == []