Fetch a posting's text on demand, following a Jobindex ad to the posting itself

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:26:20 +02:00
1 parent d168d9eb6c
commit b98561f64a
3 files changed
+307 -4

No files matched your search

+5 -4
View File
@@ -13,6 +13,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off
discover.py which ATS a company's careers site runs on, from its pages
posting.py one role's posting page fetched as text, on demand
reach.py which countries a pull asks each big board for, from the agents
"""
@@ -20,8 +21,8 @@ from __future__ import annotations
from typing import Any
from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages,
summarise)
from app.jobs import (cascade, discover, embed, extract, posting, pull, reach, score, sources,
stages, summarise)
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -40,5 +41,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
return config, Store(config.database)
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
"reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "posting",
"pull", "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
+136
View File
@@ -0,0 +1,136 @@
"""A role's posting page, fetched once as plain text on demand.
`fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's
link, keep the text as `article_text`. It is on demand, never in the pull,
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
and one click for a posting somebody opened is fair.
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
ad and a "Se jobbet" button to the posting itself: the employer's site, The
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
`posting_url` and scraped for the text; the `vis-job` page's own text is the
fallback when there is no link or it gives nothing. A stored `posting_url` is
followed directly the next time, so the `vis-job` page is not fetched twice
for the same text.
Everything that stops the fetch is a `PostingError` with the HTTP status and
message the route answers with.
"""
from __future__ import annotations
from app.models import Role
from app.storage import Store
from desk.tools import scrape
from desk.tools.checks import Page
from plugins.readers import jobindex
#: What the read page toasts when a posting page gave nothing readable, by
#: `desk.tools.failures` category.
UNREADABLE = {
"timeout": "The posting page took too long to answer. Try again in a minute.",
"unreachable": "The posting page could not be reached.",
"rate_limited": "The posting site is refusing requests for now. Try again later.",
"server_error": "The posting site answered with an error. Try again later.",
"not_found": "The posting page is gone, so the role has probably closed.",
"denied": "The posting site refused to show the page.",
"consent_wall": "The posting page showed a consent banner instead of the posting.",
"js_required": "The posting page only renders in a browser. Open it with the link instead.",
"empty": "The posting page had no readable text on it.",
}
NOTHING_READABLE = "Nothing readable came back from the posting page."
class PostingError(Exception):
"""A posting text that was not fetched. `status` is the HTTP status the
route answers with, `str(error)` the message."""
status = 422
class UnknownRole(PostingError):
status = 404
class AlreadyFetched(PostingError):
status = 409
class NoLink(PostingError):
"""The role has no link to fetch the posting from."""
class Unreadable(PostingError):
"""The page answered with nothing readable; `failure` is its
`desk.tools.failures` category, '' when none was named."""
def __init__(self, failure: str = "") -> None:
super().__init__(UNREADABLE.get(failure, NOTHING_READABLE))
self.failure = failure
def _plain(got: scrape.Scraped) -> str:
return got.content.plain_text.strip() if got.content is not None else ""
def _keeping_html(url: str) -> tuple[scrape.Scraped, str]:
"""`scrape.scrape` over one request, and the HTML that request answered
with, '' when none did."""
pages: list[str] = []
def fetch(target: str) -> Page | None:
page = scrape.HTTP.fetch(target)
if page is not None:
pages.append(page.html)
return page
got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)})
return got, pages[0] if pages else ""
def _is_jobindex(role: Role) -> bool:
return role.raw.get("provider") == "jobindex"
def _jobindex_text(store: Store, role: Role) -> str:
"""The text of a Jobindex role: the page its "Se jobbet" button leads to,
else the `vis-job` page's own. Raises `Unreadable` when neither reads."""
own: scrape.Scraped | None = None
link, failure = role.posting_url, ""
if not link:
own, html = _keeping_html(role.original_url)
link = jobindex.posting_link(html)
if link:
store.set_posting_url(role.uid, link)
if link and link != role.original_url:
got = scrape.scrape(link)
if text := _plain(got):
return text
failure = got.failure
own = own or scrape.scrape(role.original_url)
if text := _plain(own):
return text
raise Unreadable(failure or own.failure)
def fetch_posting(store: Store, uid: str) -> Role:
"""Fetch the posting page once and keep it as plain text, for a role
whose description is a teaser. Synchronous: one request the person asked
for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`."""
role = store.role(uid)
if role is None:
raise UnknownRole(f"no such role: {uid}")
if role.article_text:
raise AlreadyFetched("The posting text is already fetched.")
if not role.original_url:
raise NoLink("This role has no link to fetch the posting from.")
if _is_jobindex(role):
text = _jobindex_text(store, role)
else:
got = scrape.scrape(role.original_url)
if not (text := _plain(got)):
raise Unreadable(got.failure)
updated = store.set_article_text(uid, text)
if updated is None:
raise UnknownRole(f"no such role: {uid}")
return updated
+166
View File
@@ -0,0 +1,166 @@
"""`fetch_posting`: a role's posting text on demand, and for a Jobindex role
the page its "Se jobbet" button leads to. No network: `scrape` is stubbed."""
from __future__ import annotations
from datetime import UTC, datetime
from pathlib import Path
import pytest
from app.jobs import posting
from app.storage import Store
from desk.models import Item
from desk.tools import scrape
from desk.tools.article import ArticleContent
from desk.tools.checks import Page
from prepped import upsert
ROOT = Path(__file__).resolve().parents[1]
VIS_JOB = "https://www.jobindex.dk/vis-job/h1702752"
LINK = ("https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring"
"?promotion=2220670-jobindex")
PAGE = (ROOT / "tests" / "fixtures" / "jobindex" / "vis-job-employer.html").read_text("utf-8")
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def role(store: Store, url: str = "https://boards.example/acme/1", source_id: str = "1",
**raw: object) -> str:
item = Item(source_id=source_id, location=url, title="Engineer", body="A teaser.",
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
item.source = "ats"
upsert(store, item)
return item.uid
def jobindex_role(store: Store) -> str:
return role(store, VIS_JOB, provider="jobindex")
def content(text: str, url: str = "") -> scrape.Scraped:
return scrape.Scraped(ArticleContent(format="plain", content=text, source_url=url,
method="test"), "", "http", ())
def failed(category: str) -> scrape.Scraped:
return scrape.Scraped(None, category, "http", ())
class Web:
"""Stands in for the network: `pages` maps a URL to what `scrape.HTTP`
answers with, `texts` a URL to what the article of that page reads as."""
def __init__(self, monkeypatch: pytest.MonkeyPatch, pages: dict[str, str],
texts: dict[str, str]) -> None:
self.asked: list[str] = []
self.pages, self.texts = pages, texts
monkeypatch.setattr(scrape, "HTTP", scrape.Getter("http", self.fetch))
monkeypatch.setattr(scrape, "STANDARD", {"http": scrape.HTTP})
monkeypatch.setattr(scrape, "scrape", self.scrape)
def fetch(self, url: str) -> Page | None:
self.asked.append(url)
html = self.pages.get(url)
return None if html is None else Page(status=200, html=html, url=url, headers={})
def scrape(self, url: str, getters: object = None, first: str = "http") -> scrape.Scraped:
if getters is not None: # the page is fetched by the getter it was handed
page = getters["http"].fetch(url) # type: ignore[index]
return content(self.texts[url], url) if page and url in self.texts else failed("empty")
self.asked.append(url)
return content(self.texts[url], url) if url in self.texts else failed("not_found")
def test_a_role_that_cannot_be_fetched_says_why(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
Web(monkeypatch, {}, {})
with pytest.raises(posting.UnknownRole) as unknown:
posting.fetch_posting(store, "nope")
assert (unknown.value.status, str(unknown.value)) == (404, "no such role: nope")
bare = role(store, url="")
with pytest.raises(posting.NoLink) as nolink:
posting.fetch_posting(store, bare)
assert nolink.value.status == 422
def test_an_ordinary_role_is_scraped_from_its_link_once(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
uid = role(store)
web = Web(monkeypatch, {}, {})
with pytest.raises(posting.Unreadable) as gone:
posting.fetch_posting(store, uid)
assert gone.value.status == 422 and "gone" in str(gone.value)
assert web.asked == ["https://boards.example/acme/1"]
web.texts["https://boards.example/acme/1"] = " The whole posting.\n"
got = posting.fetch_posting(store, uid)
assert got.article_text == "The whole posting." and got.article_at
assert got.posting_url == ""
with pytest.raises(posting.AlreadyFetched) as again:
posting.fetch_posting(store, uid)
assert again.value.status == 409
def test_an_unknown_failure_reads_as_nothing_readable(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
uid = role(store)
web = Web(monkeypatch, {}, {})
monkeypatch.setattr(web, "scrape", lambda url, *a, **k: failed("weird"))
monkeypatch.setattr(scrape, "scrape", web.scrape)
with pytest.raises(posting.Unreadable, match="Nothing readable"):
posting.fetch_posting(store, uid)
def test_a_jobindex_role_follows_its_se_jobbet_link_and_keeps_it(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad.", LINK: "The real posting."})
got = posting.fetch_posting(store, uid)
assert got.article_text == "The real posting."
assert got.posting_url == LINK and got.original_url == VIS_JOB
assert web.asked == [VIS_JOB, LINK]
stored = store.role(uid)
assert stored is not None and stored.posting_url == LINK
def test_a_jobindex_role_falls_back_to_the_ads_own_text(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
# The link gives nothing readable: the link is kept, the vis-job text stored.
uid = jobindex_role(store)
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad."})
got = posting.fetch_posting(store, uid)
assert got.article_text == "The Jobindex ad." and got.posting_url == LINK
assert web.asked == [VIS_JOB, LINK]
# No button on the page: the ad's own text, no link, one request.
other = role(store, "https://www.jobindex.dk/vis-job/r2", "2", provider="jobindex")
web = Web(monkeypatch, {"https://www.jobindex.dk/vis-job/r2": "<p>Ingen knap</p>"},
{"https://www.jobindex.dk/vis-job/r2": "Ad two."})
got = posting.fetch_posting(store, other)
assert got.article_text == "Ad two." and got.posting_url == ""
assert web.asked == ["https://www.jobindex.dk/vis-job/r2"]
def test_a_jobindex_role_whose_pages_read_as_nothing_is_unreadable_but_keeps_the_link(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
Web(monkeypatch, {VIS_JOB: PAGE}, {})
with pytest.raises(posting.Unreadable) as error:
posting.fetch_posting(store, uid)
assert error.value.failure == "not_found"
stored = store.role(uid)
assert stored is not None and stored.posting_url == LINK and stored.article_text == ""
def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
store.set_posting_url(uid, LINK)
web = Web(monkeypatch, {}, {LINK: "The real posting."})
assert posting.fetch_posting(store, uid).article_text == "The real posting."
assert web.asked == [LINK]