diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index ec9bd9e..dc078d9 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -13,6 +13,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. sources.py what a pull reads, changed by hand: boards and searches added, switched on or off discover.py which ATS a company's careers site runs on, from its pages + posting.py one role's posting page fetched as text, on demand reach.py which countries a pull asks each big board for, from the agents """ @@ -20,8 +21,8 @@ from __future__ import annotations from typing import Any -from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages, - summarise) +from app.jobs import (cascade, discover, embed, extract, posting, pull, reach, score, sources, + stages, summarise) from app.jobs.pull import search_target, seed_searches from app.jobs.score import seed_agents @@ -40,5 +41,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]: return config, Store(config.database) -__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull", - "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"] +__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "posting", + "pull", "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"] diff --git a/app/jobs/posting.py b/app/jobs/posting.py new file mode 100644 index 0000000..9d5fc2a --- /dev/null +++ b/app/jobs/posting.py @@ -0,0 +1,136 @@ +"""A role's posting page, fetched once as plain text on demand. + +`fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's +link, keep the text as `article_text`. It is on demand, never in the pull, +because a fetch of a Jobindex page counts as a click in Jobindex's statistics +and one click for a posting somebody opened is fair. + +A Jobindex role links to Jobindex's own `vis-job` page, which carries a short +ad and a "Se jobbet" button to the posting itself: the employer's site, The +Hub or Jobindex's `jobannonce`. That button's link is found, kept as +`posting_url` and scraped for the text; the `vis-job` page's own text is the +fallback when there is no link or it gives nothing. A stored `posting_url` is +followed directly the next time, so the `vis-job` page is not fetched twice +for the same text. + +Everything that stops the fetch is a `PostingError` with the HTTP status and +message the route answers with. +""" + +from __future__ import annotations + +from app.models import Role +from app.storage import Store +from desk.tools import scrape +from desk.tools.checks import Page +from plugins.readers import jobindex + +#: What the read page toasts when a posting page gave nothing readable, by +#: `desk.tools.failures` category. +UNREADABLE = { + "timeout": "The posting page took too long to answer. Try again in a minute.", + "unreachable": "The posting page could not be reached.", + "rate_limited": "The posting site is refusing requests for now. Try again later.", + "server_error": "The posting site answered with an error. Try again later.", + "not_found": "The posting page is gone, so the role has probably closed.", + "denied": "The posting site refused to show the page.", + "consent_wall": "The posting page showed a consent banner instead of the posting.", + "js_required": "The posting page only renders in a browser. Open it with the link instead.", + "empty": "The posting page had no readable text on it.", +} +NOTHING_READABLE = "Nothing readable came back from the posting page." + + +class PostingError(Exception): + """A posting text that was not fetched. `status` is the HTTP status the + route answers with, `str(error)` the message.""" + + status = 422 + + +class UnknownRole(PostingError): + status = 404 + + +class AlreadyFetched(PostingError): + status = 409 + + +class NoLink(PostingError): + """The role has no link to fetch the posting from.""" + + +class Unreadable(PostingError): + """The page answered with nothing readable; `failure` is its + `desk.tools.failures` category, '' when none was named.""" + + def __init__(self, failure: str = "") -> None: + super().__init__(UNREADABLE.get(failure, NOTHING_READABLE)) + self.failure = failure + + +def _plain(got: scrape.Scraped) -> str: + return got.content.plain_text.strip() if got.content is not None else "" + + +def _keeping_html(url: str) -> tuple[scrape.Scraped, str]: + """`scrape.scrape` over one request, and the HTML that request answered + with, '' when none did.""" + pages: list[str] = [] + + def fetch(target: str) -> Page | None: + page = scrape.HTTP.fetch(target) + if page is not None: + pages.append(page.html) + return page + + got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)}) + return got, pages[0] if pages else "" + + +def _is_jobindex(role: Role) -> bool: + return role.raw.get("provider") == "jobindex" + + +def _jobindex_text(store: Store, role: Role) -> str: + """The text of a Jobindex role: the page its "Se jobbet" button leads to, + else the `vis-job` page's own. Raises `Unreadable` when neither reads.""" + own: scrape.Scraped | None = None + link, failure = role.posting_url, "" + if not link: + own, html = _keeping_html(role.original_url) + link = jobindex.posting_link(html) + if link: + store.set_posting_url(role.uid, link) + if link and link != role.original_url: + got = scrape.scrape(link) + if text := _plain(got): + return text + failure = got.failure + own = own or scrape.scrape(role.original_url) + if text := _plain(own): + return text + raise Unreadable(failure or own.failure) + + +def fetch_posting(store: Store, uid: str) -> Role: + """Fetch the posting page once and keep it as plain text, for a role + whose description is a teaser. Synchronous: one request the person asked + for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`.""" + role = store.role(uid) + if role is None: + raise UnknownRole(f"no such role: {uid}") + if role.article_text: + raise AlreadyFetched("The posting text is already fetched.") + if not role.original_url: + raise NoLink("This role has no link to fetch the posting from.") + if _is_jobindex(role): + text = _jobindex_text(store, role) + else: + got = scrape.scrape(role.original_url) + if not (text := _plain(got)): + raise Unreadable(got.failure) + updated = store.set_article_text(uid, text) + if updated is None: + raise UnknownRole(f"no such role: {uid}") + return updated diff --git a/tests/test_posting.py b/tests/test_posting.py new file mode 100644 index 0000000..f4ffcc6 --- /dev/null +++ b/tests/test_posting.py @@ -0,0 +1,166 @@ +"""`fetch_posting`: a role's posting text on demand, and for a Jobindex role +the page its "Se jobbet" button leads to. No network: `scrape` is stubbed.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from app.jobs import posting +from app.storage import Store +from desk.models import Item +from desk.tools import scrape +from desk.tools.article import ArticleContent +from desk.tools.checks import Page +from prepped import upsert + +ROOT = Path(__file__).resolve().parents[1] +VIS_JOB = "https://www.jobindex.dk/vis-job/h1702752" +LINK = ("https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring" + "?promotion=2220670-jobindex") +PAGE = (ROOT / "tests" / "fixtures" / "jobindex" / "vis-job-employer.html").read_text("utf-8") + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +def role(store: Store, url: str = "https://boards.example/acme/1", source_id: str = "1", + **raw: object) -> str: + item = Item(source_id=source_id, location=url, title="Engineer", body="A teaser.", + published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw)) + item.source = "ats" + upsert(store, item) + return item.uid + + +def jobindex_role(store: Store) -> str: + return role(store, VIS_JOB, provider="jobindex") + + +def content(text: str, url: str = "") -> scrape.Scraped: + return scrape.Scraped(ArticleContent(format="plain", content=text, source_url=url, + method="test"), "", "http", ()) + + +def failed(category: str) -> scrape.Scraped: + return scrape.Scraped(None, category, "http", ()) + + +class Web: + """Stands in for the network: `pages` maps a URL to what `scrape.HTTP` + answers with, `texts` a URL to what the article of that page reads as.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, pages: dict[str, str], + texts: dict[str, str]) -> None: + self.asked: list[str] = [] + self.pages, self.texts = pages, texts + monkeypatch.setattr(scrape, "HTTP", scrape.Getter("http", self.fetch)) + monkeypatch.setattr(scrape, "STANDARD", {"http": scrape.HTTP}) + monkeypatch.setattr(scrape, "scrape", self.scrape) + + def fetch(self, url: str) -> Page | None: + self.asked.append(url) + html = self.pages.get(url) + return None if html is None else Page(status=200, html=html, url=url, headers={}) + + def scrape(self, url: str, getters: object = None, first: str = "http") -> scrape.Scraped: + if getters is not None: # the page is fetched by the getter it was handed + page = getters["http"].fetch(url) # type: ignore[index] + return content(self.texts[url], url) if page and url in self.texts else failed("empty") + self.asked.append(url) + return content(self.texts[url], url) if url in self.texts else failed("not_found") + + +def test_a_role_that_cannot_be_fetched_says_why(store: Store, + monkeypatch: pytest.MonkeyPatch) -> None: + Web(monkeypatch, {}, {}) + with pytest.raises(posting.UnknownRole) as unknown: + posting.fetch_posting(store, "nope") + assert (unknown.value.status, str(unknown.value)) == (404, "no such role: nope") + + bare = role(store, url="") + with pytest.raises(posting.NoLink) as nolink: + posting.fetch_posting(store, bare) + assert nolink.value.status == 422 + + +def test_an_ordinary_role_is_scraped_from_its_link_once(store: Store, + monkeypatch: pytest.MonkeyPatch) -> None: + uid = role(store) + web = Web(monkeypatch, {}, {}) + with pytest.raises(posting.Unreadable) as gone: + posting.fetch_posting(store, uid) + assert gone.value.status == 422 and "gone" in str(gone.value) + assert web.asked == ["https://boards.example/acme/1"] + + web.texts["https://boards.example/acme/1"] = " The whole posting.\n" + got = posting.fetch_posting(store, uid) + assert got.article_text == "The whole posting." and got.article_at + assert got.posting_url == "" + with pytest.raises(posting.AlreadyFetched) as again: + posting.fetch_posting(store, uid) + assert again.value.status == 409 + + +def test_an_unknown_failure_reads_as_nothing_readable(store: Store, + monkeypatch: pytest.MonkeyPatch) -> None: + uid = role(store) + web = Web(monkeypatch, {}, {}) + monkeypatch.setattr(web, "scrape", lambda url, *a, **k: failed("weird")) + monkeypatch.setattr(scrape, "scrape", web.scrape) + with pytest.raises(posting.Unreadable, match="Nothing readable"): + posting.fetch_posting(store, uid) + + +def test_a_jobindex_role_follows_its_se_jobbet_link_and_keeps_it( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + uid = jobindex_role(store) + web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad.", LINK: "The real posting."}) + got = posting.fetch_posting(store, uid) + assert got.article_text == "The real posting." + assert got.posting_url == LINK and got.original_url == VIS_JOB + assert web.asked == [VIS_JOB, LINK] + stored = store.role(uid) + assert stored is not None and stored.posting_url == LINK + + +def test_a_jobindex_role_falls_back_to_the_ads_own_text( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + # The link gives nothing readable: the link is kept, the vis-job text stored. + uid = jobindex_role(store) + web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad."}) + got = posting.fetch_posting(store, uid) + assert got.article_text == "The Jobindex ad." and got.posting_url == LINK + assert web.asked == [VIS_JOB, LINK] + + # No button on the page: the ad's own text, no link, one request. + other = role(store, "https://www.jobindex.dk/vis-job/r2", "2", provider="jobindex") + web = Web(monkeypatch, {"https://www.jobindex.dk/vis-job/r2": "
Ingen knap
"}, + {"https://www.jobindex.dk/vis-job/r2": "Ad two."}) + got = posting.fetch_posting(store, other) + assert got.article_text == "Ad two." and got.posting_url == "" + assert web.asked == ["https://www.jobindex.dk/vis-job/r2"] + + +def test_a_jobindex_role_whose_pages_read_as_nothing_is_unreadable_but_keeps_the_link( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + uid = jobindex_role(store) + Web(monkeypatch, {VIS_JOB: PAGE}, {}) + with pytest.raises(posting.Unreadable) as error: + posting.fetch_posting(store, uid) + assert error.value.failure == "not_found" + stored = store.role(uid) + assert stored is not None and stored.posting_url == LINK and stored.article_text == "" + + +def test_a_kept_link_is_followed_without_fetching_the_ad_page_again( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + uid = jobindex_role(store) + store.set_posting_url(uid, LINK) + web = Web(monkeypatch, {}, {LINK: "The real posting."}) + assert posting.fetch_posting(store, uid).article_text == "The real posting." + assert web.asked == [LINK]