Fetch a posting's text on demand, following a Jobindex ad to the posting itself
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d168d9eb6c
commit
b98561f64a
3 files changed
+307
-4
No files matched your search
@@ -13,6 +13,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
|||||||
sources.py what a pull reads, changed by hand: boards and searches
|
sources.py what a pull reads, changed by hand: boards and searches
|
||||||
added, switched on or off
|
added, switched on or off
|
||||||
discover.py which ATS a company's careers site runs on, from its pages
|
discover.py which ATS a company's careers site runs on, from its pages
|
||||||
|
posting.py one role's posting page fetched as text, on demand
|
||||||
reach.py which countries a pull asks each big board for, from the agents
|
reach.py which countries a pull asks each big board for, from the agents
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -20,8 +21,8 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages,
|
from app.jobs import (cascade, discover, embed, extract, posting, pull, reach, score, sources,
|
||||||
summarise)
|
stages, summarise)
|
||||||
from app.jobs.pull import search_target, seed_searches
|
from app.jobs.pull import search_target, seed_searches
|
||||||
from app.jobs.score import seed_agents
|
from app.jobs.score import seed_agents
|
||||||
|
|
||||||
@@ -40,5 +41,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
|
|||||||
return config, Store(config.database)
|
return config, Store(config.database)
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
|
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "posting",
|
||||||
"reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
|
"pull", "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
"""A role's posting page, fetched once as plain text on demand.
|
||||||
|
|
||||||
|
`fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's
|
||||||
|
link, keep the text as `article_text`. It is on demand, never in the pull,
|
||||||
|
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
|
||||||
|
and one click for a posting somebody opened is fair.
|
||||||
|
|
||||||
|
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
|
||||||
|
ad and a "Se jobbet" button to the posting itself: the employer's site, The
|
||||||
|
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
|
||||||
|
`posting_url` and scraped for the text; the `vis-job` page's own text is the
|
||||||
|
fallback when there is no link or it gives nothing. A stored `posting_url` is
|
||||||
|
followed directly the next time, so the `vis-job` page is not fetched twice
|
||||||
|
for the same text.
|
||||||
|
|
||||||
|
Everything that stops the fetch is a `PostingError` with the HTTP status and
|
||||||
|
message the route answers with.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.models import Role
|
||||||
|
from app.storage import Store
|
||||||
|
from desk.tools import scrape
|
||||||
|
from desk.tools.checks import Page
|
||||||
|
from plugins.readers import jobindex
|
||||||
|
|
||||||
|
#: What the read page toasts when a posting page gave nothing readable, by
|
||||||
|
#: `desk.tools.failures` category.
|
||||||
|
UNREADABLE = {
|
||||||
|
"timeout": "The posting page took too long to answer. Try again in a minute.",
|
||||||
|
"unreachable": "The posting page could not be reached.",
|
||||||
|
"rate_limited": "The posting site is refusing requests for now. Try again later.",
|
||||||
|
"server_error": "The posting site answered with an error. Try again later.",
|
||||||
|
"not_found": "The posting page is gone, so the role has probably closed.",
|
||||||
|
"denied": "The posting site refused to show the page.",
|
||||||
|
"consent_wall": "The posting page showed a consent banner instead of the posting.",
|
||||||
|
"js_required": "The posting page only renders in a browser. Open it with the link instead.",
|
||||||
|
"empty": "The posting page had no readable text on it.",
|
||||||
|
}
|
||||||
|
NOTHING_READABLE = "Nothing readable came back from the posting page."
|
||||||
|
|
||||||
|
|
||||||
|
class PostingError(Exception):
|
||||||
|
"""A posting text that was not fetched. `status` is the HTTP status the
|
||||||
|
route answers with, `str(error)` the message."""
|
||||||
|
|
||||||
|
status = 422
|
||||||
|
|
||||||
|
|
||||||
|
class UnknownRole(PostingError):
|
||||||
|
status = 404
|
||||||
|
|
||||||
|
|
||||||
|
class AlreadyFetched(PostingError):
|
||||||
|
status = 409
|
||||||
|
|
||||||
|
|
||||||
|
class NoLink(PostingError):
|
||||||
|
"""The role has no link to fetch the posting from."""
|
||||||
|
|
||||||
|
|
||||||
|
class Unreadable(PostingError):
|
||||||
|
"""The page answered with nothing readable; `failure` is its
|
||||||
|
`desk.tools.failures` category, '' when none was named."""
|
||||||
|
|
||||||
|
def __init__(self, failure: str = "") -> None:
|
||||||
|
super().__init__(UNREADABLE.get(failure, NOTHING_READABLE))
|
||||||
|
self.failure = failure
|
||||||
|
|
||||||
|
|
||||||
|
def _plain(got: scrape.Scraped) -> str:
|
||||||
|
return got.content.plain_text.strip() if got.content is not None else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _keeping_html(url: str) -> tuple[scrape.Scraped, str]:
|
||||||
|
"""`scrape.scrape` over one request, and the HTML that request answered
|
||||||
|
with, '' when none did."""
|
||||||
|
pages: list[str] = []
|
||||||
|
|
||||||
|
def fetch(target: str) -> Page | None:
|
||||||
|
page = scrape.HTTP.fetch(target)
|
||||||
|
if page is not None:
|
||||||
|
pages.append(page.html)
|
||||||
|
return page
|
||||||
|
|
||||||
|
got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)})
|
||||||
|
return got, pages[0] if pages else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _is_jobindex(role: Role) -> bool:
|
||||||
|
return role.raw.get("provider") == "jobindex"
|
||||||
|
|
||||||
|
|
||||||
|
def _jobindex_text(store: Store, role: Role) -> str:
|
||||||
|
"""The text of a Jobindex role: the page its "Se jobbet" button leads to,
|
||||||
|
else the `vis-job` page's own. Raises `Unreadable` when neither reads."""
|
||||||
|
own: scrape.Scraped | None = None
|
||||||
|
link, failure = role.posting_url, ""
|
||||||
|
if not link:
|
||||||
|
own, html = _keeping_html(role.original_url)
|
||||||
|
link = jobindex.posting_link(html)
|
||||||
|
if link:
|
||||||
|
store.set_posting_url(role.uid, link)
|
||||||
|
if link and link != role.original_url:
|
||||||
|
got = scrape.scrape(link)
|
||||||
|
if text := _plain(got):
|
||||||
|
return text
|
||||||
|
failure = got.failure
|
||||||
|
own = own or scrape.scrape(role.original_url)
|
||||||
|
if text := _plain(own):
|
||||||
|
return text
|
||||||
|
raise Unreadable(failure or own.failure)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_posting(store: Store, uid: str) -> Role:
|
||||||
|
"""Fetch the posting page once and keep it as plain text, for a role
|
||||||
|
whose description is a teaser. Synchronous: one request the person asked
|
||||||
|
for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`."""
|
||||||
|
role = store.role(uid)
|
||||||
|
if role is None:
|
||||||
|
raise UnknownRole(f"no such role: {uid}")
|
||||||
|
if role.article_text:
|
||||||
|
raise AlreadyFetched("The posting text is already fetched.")
|
||||||
|
if not role.original_url:
|
||||||
|
raise NoLink("This role has no link to fetch the posting from.")
|
||||||
|
if _is_jobindex(role):
|
||||||
|
text = _jobindex_text(store, role)
|
||||||
|
else:
|
||||||
|
got = scrape.scrape(role.original_url)
|
||||||
|
if not (text := _plain(got)):
|
||||||
|
raise Unreadable(got.failure)
|
||||||
|
updated = store.set_article_text(uid, text)
|
||||||
|
if updated is None:
|
||||||
|
raise UnknownRole(f"no such role: {uid}")
|
||||||
|
return updated
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
"""`fetch_posting`: a role's posting text on demand, and for a Jobindex role
|
||||||
|
the page its "Se jobbet" button leads to. No network: `scrape` is stubbed."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.jobs import posting
|
||||||
|
from app.storage import Store
|
||||||
|
from desk.models import Item
|
||||||
|
from desk.tools import scrape
|
||||||
|
from desk.tools.article import ArticleContent
|
||||||
|
from desk.tools.checks import Page
|
||||||
|
from prepped import upsert
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
VIS_JOB = "https://www.jobindex.dk/vis-job/h1702752"
|
||||||
|
LINK = ("https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring"
|
||||||
|
"?promotion=2220670-jobindex")
|
||||||
|
PAGE = (ROOT / "tests" / "fixtures" / "jobindex" / "vis-job-employer.html").read_text("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def store(tmp_path: Path) -> Store:
|
||||||
|
return Store(tmp_path / "roledesk.db")
|
||||||
|
|
||||||
|
|
||||||
|
def role(store: Store, url: str = "https://boards.example/acme/1", source_id: str = "1",
|
||||||
|
**raw: object) -> str:
|
||||||
|
item = Item(source_id=source_id, location=url, title="Engineer", body="A teaser.",
|
||||||
|
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
|
||||||
|
item.source = "ats"
|
||||||
|
upsert(store, item)
|
||||||
|
return item.uid
|
||||||
|
|
||||||
|
|
||||||
|
def jobindex_role(store: Store) -> str:
|
||||||
|
return role(store, VIS_JOB, provider="jobindex")
|
||||||
|
|
||||||
|
|
||||||
|
def content(text: str, url: str = "") -> scrape.Scraped:
|
||||||
|
return scrape.Scraped(ArticleContent(format="plain", content=text, source_url=url,
|
||||||
|
method="test"), "", "http", ())
|
||||||
|
|
||||||
|
|
||||||
|
def failed(category: str) -> scrape.Scraped:
|
||||||
|
return scrape.Scraped(None, category, "http", ())
|
||||||
|
|
||||||
|
|
||||||
|
class Web:
|
||||||
|
"""Stands in for the network: `pages` maps a URL to what `scrape.HTTP`
|
||||||
|
answers with, `texts` a URL to what the article of that page reads as."""
|
||||||
|
|
||||||
|
def __init__(self, monkeypatch: pytest.MonkeyPatch, pages: dict[str, str],
|
||||||
|
texts: dict[str, str]) -> None:
|
||||||
|
self.asked: list[str] = []
|
||||||
|
self.pages, self.texts = pages, texts
|
||||||
|
monkeypatch.setattr(scrape, "HTTP", scrape.Getter("http", self.fetch))
|
||||||
|
monkeypatch.setattr(scrape, "STANDARD", {"http": scrape.HTTP})
|
||||||
|
monkeypatch.setattr(scrape, "scrape", self.scrape)
|
||||||
|
|
||||||
|
def fetch(self, url: str) -> Page | None:
|
||||||
|
self.asked.append(url)
|
||||||
|
html = self.pages.get(url)
|
||||||
|
return None if html is None else Page(status=200, html=html, url=url, headers={})
|
||||||
|
|
||||||
|
def scrape(self, url: str, getters: object = None, first: str = "http") -> scrape.Scraped:
|
||||||
|
if getters is not None: # the page is fetched by the getter it was handed
|
||||||
|
page = getters["http"].fetch(url) # type: ignore[index]
|
||||||
|
return content(self.texts[url], url) if page and url in self.texts else failed("empty")
|
||||||
|
self.asked.append(url)
|
||||||
|
return content(self.texts[url], url) if url in self.texts else failed("not_found")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_that_cannot_be_fetched_says_why(store: Store,
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
Web(monkeypatch, {}, {})
|
||||||
|
with pytest.raises(posting.UnknownRole) as unknown:
|
||||||
|
posting.fetch_posting(store, "nope")
|
||||||
|
assert (unknown.value.status, str(unknown.value)) == (404, "no such role: nope")
|
||||||
|
|
||||||
|
bare = role(store, url="")
|
||||||
|
with pytest.raises(posting.NoLink) as nolink:
|
||||||
|
posting.fetch_posting(store, bare)
|
||||||
|
assert nolink.value.status == 422
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_ordinary_role_is_scraped_from_its_link_once(store: Store,
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
uid = role(store)
|
||||||
|
web = Web(monkeypatch, {}, {})
|
||||||
|
with pytest.raises(posting.Unreadable) as gone:
|
||||||
|
posting.fetch_posting(store, uid)
|
||||||
|
assert gone.value.status == 422 and "gone" in str(gone.value)
|
||||||
|
assert web.asked == ["https://boards.example/acme/1"]
|
||||||
|
|
||||||
|
web.texts["https://boards.example/acme/1"] = " The whole posting.\n"
|
||||||
|
got = posting.fetch_posting(store, uid)
|
||||||
|
assert got.article_text == "The whole posting." and got.article_at
|
||||||
|
assert got.posting_url == ""
|
||||||
|
with pytest.raises(posting.AlreadyFetched) as again:
|
||||||
|
posting.fetch_posting(store, uid)
|
||||||
|
assert again.value.status == 409
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unknown_failure_reads_as_nothing_readable(store: Store,
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
uid = role(store)
|
||||||
|
web = Web(monkeypatch, {}, {})
|
||||||
|
monkeypatch.setattr(web, "scrape", lambda url, *a, **k: failed("weird"))
|
||||||
|
monkeypatch.setattr(scrape, "scrape", web.scrape)
|
||||||
|
with pytest.raises(posting.Unreadable, match="Nothing readable"):
|
||||||
|
posting.fetch_posting(store, uid)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_jobindex_role_follows_its_se_jobbet_link_and_keeps_it(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
uid = jobindex_role(store)
|
||||||
|
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad.", LINK: "The real posting."})
|
||||||
|
got = posting.fetch_posting(store, uid)
|
||||||
|
assert got.article_text == "The real posting."
|
||||||
|
assert got.posting_url == LINK and got.original_url == VIS_JOB
|
||||||
|
assert web.asked == [VIS_JOB, LINK]
|
||||||
|
stored = store.role(uid)
|
||||||
|
assert stored is not None and stored.posting_url == LINK
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_jobindex_role_falls_back_to_the_ads_own_text(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
# The link gives nothing readable: the link is kept, the vis-job text stored.
|
||||||
|
uid = jobindex_role(store)
|
||||||
|
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad."})
|
||||||
|
got = posting.fetch_posting(store, uid)
|
||||||
|
assert got.article_text == "The Jobindex ad." and got.posting_url == LINK
|
||||||
|
assert web.asked == [VIS_JOB, LINK]
|
||||||
|
|
||||||
|
# No button on the page: the ad's own text, no link, one request.
|
||||||
|
other = role(store, "https://www.jobindex.dk/vis-job/r2", "2", provider="jobindex")
|
||||||
|
web = Web(monkeypatch, {"https://www.jobindex.dk/vis-job/r2": "<p>Ingen knap</p>"},
|
||||||
|
{"https://www.jobindex.dk/vis-job/r2": "Ad two."})
|
||||||
|
got = posting.fetch_posting(store, other)
|
||||||
|
assert got.article_text == "Ad two." and got.posting_url == ""
|
||||||
|
assert web.asked == ["https://www.jobindex.dk/vis-job/r2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_jobindex_role_whose_pages_read_as_nothing_is_unreadable_but_keeps_the_link(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
uid = jobindex_role(store)
|
||||||
|
Web(monkeypatch, {VIS_JOB: PAGE}, {})
|
||||||
|
with pytest.raises(posting.Unreadable) as error:
|
||||||
|
posting.fetch_posting(store, uid)
|
||||||
|
assert error.value.failure == "not_found"
|
||||||
|
stored = store.role(uid)
|
||||||
|
assert stored is not None and stored.posting_url == LINK and stored.article_text == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
uid = jobindex_role(store)
|
||||||
|
store.set_posting_url(uid, LINK)
|
||||||
|
web = Web(monkeypatch, {}, {LINK: "The real posting."})
|
||||||
|
assert posting.fetch_posting(store, uid).article_text == "The real posting."
|
||||||
|
assert web.asked == [LINK]
|
||||||
Reference in new issue
Block a user