diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index ec9bd9e..dc078d9 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -13,6 +13,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. sources.py what a pull reads, changed by hand: boards and searches added, switched on or off discover.py which ATS a company's careers site runs on, from its pages + posting.py one role's posting page fetched as text, on demand reach.py which countries a pull asks each big board for, from the agents """ @@ -20,8 +21,8 @@ from __future__ import annotations from typing import Any -from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages, - summarise) +from app.jobs import (cascade, discover, embed, extract, posting, pull, reach, score, sources, + stages, summarise) from app.jobs.pull import search_target, seed_searches from app.jobs.score import seed_agents @@ -40,5 +41,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]: return config, Store(config.database) -__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull", - "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"] +__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "posting", + "pull", "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"] diff --git a/app/jobs/posting.py b/app/jobs/posting.py new file mode 100644 index 0000000..9d5fc2a --- /dev/null +++ b/app/jobs/posting.py @@ -0,0 +1,136 @@ +"""A role's posting page, fetched once as plain text on demand. + +`fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's +link, keep the text as `article_text`. It is on demand, never in the pull, +because a fetch of a Jobindex page counts as a click in Jobindex's statistics +and one click for a posting somebody opened is fair. + +A Jobindex role links to Jobindex's own `vis-job` page, which carries a short +ad and a "Se jobbet" button to the posting itself: the employer's site, The +Hub or Jobindex's `jobannonce`. That button's link is found, kept as +`posting_url` and scraped for the text; the `vis-job` page's own text is the +fallback when there is no link or it gives nothing. A stored `posting_url` is +followed directly the next time, so the `vis-job` page is not fetched twice +for the same text. + +Everything that stops the fetch is a `PostingError` with the HTTP status and +message the route answers with. +""" + +from __future__ import annotations + +from app.models import Role +from app.storage import Store +from desk.tools import scrape +from desk.tools.checks import Page +from plugins.readers import jobindex + +#: What the read page toasts when a posting page gave nothing readable, by +#: `desk.tools.failures` category. +UNREADABLE = { + "timeout": "The posting page took too long to answer. Try again in a minute.", + "unreachable": "The posting page could not be reached.", + "rate_limited": "The posting site is refusing requests for now. Try again later.", + "server_error": "The posting site answered with an error. Try again later.", + "not_found": "The posting page is gone, so the role has probably closed.", + "denied": "The posting site refused to show the page.", + "consent_wall": "The posting page showed a consent banner instead of the posting.", + "js_required": "The posting page only renders in a browser. Open it with the link instead.", + "empty": "The posting page had no readable text on it.", +} +NOTHING_READABLE = "Nothing readable came back from the posting page." + + +class PostingError(Exception): + """A posting text that was not fetched. `status` is the HTTP status the + route answers with, `str(error)` the message.""" + + status = 422 + + +class UnknownRole(PostingError): + status = 404 + + +class AlreadyFetched(PostingError): + status = 409 + + +class NoLink(PostingError): + """The role has no link to fetch the posting from.""" + + +class Unreadable(PostingError): + """The page answered with nothing readable; `failure` is its + `desk.tools.failures` category, '' when none was named.""" + + def __init__(self, failure: str = "") -> None: + super().__init__(UNREADABLE.get(failure, NOTHING_READABLE)) + self.failure = failure + + +def _plain(got: scrape.Scraped) -> str: + return got.content.plain_text.strip() if got.content is not None else "" + + +def _keeping_html(url: str) -> tuple[scrape.Scraped, str]: + """`scrape.scrape` over one request, and the HTML that request answered + with, '' when none did.""" + pages: list[str] = [] + + def fetch(target: str) -> Page | None: + page = scrape.HTTP.fetch(target) + if page is not None: + pages.append(page.html) + return page + + got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)}) + return got, pages[0] if pages else "" + + +def _is_jobindex(role: Role) -> bool: + return role.raw.get("provider") == "jobindex" + + +def _jobindex_text(store: Store, role: Role) -> str: + """The text of a Jobindex role: the page its "Se jobbet" button leads to, + else the `vis-job` page's own. Raises `Unreadable` when neither reads.""" + own: scrape.Scraped | None = None + link, failure = role.posting_url, "" + if not link: + own, html = _keeping_html(role.original_url) + link = jobindex.posting_link(html) + if link: + store.set_posting_url(role.uid, link) + if link and link != role.original_url: + got = scrape.scrape(link) + if text := _plain(got): + return text + failure = got.failure + own = own or scrape.scrape(role.original_url) + if text := _plain(own): + return text + raise Unreadable(failure or own.failure) + + +def fetch_posting(store: Store, uid: str) -> Role: + """Fetch the posting page once and keep it as plain text, for a role + whose description is a teaser. Synchronous: one request the person asked + for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`.""" + role = store.role(uid) + if role is None: + raise UnknownRole(f"no such role: {uid}") + if role.article_text: + raise AlreadyFetched("The posting text is already fetched.") + if not role.original_url: + raise NoLink("This role has no link to fetch the posting from.") + if _is_jobindex(role): + text = _jobindex_text(store, role) + else: + got = scrape.scrape(role.original_url) + if not (text := _plain(got)): + raise Unreadable(got.failure) + updated = store.set_article_text(uid, text) + if updated is None: + raise UnknownRole(f"no such role: {uid}") + return updated diff --git a/app/models.py b/app/models.py index 715a986..5641a6b 100644 --- a/app/models.py +++ b/app/models.py @@ -874,6 +874,7 @@ class RoleCopy(Model): host: str = "" place: str = "" original_url: str = "" + posting_url: str = "" posted_at: str = "" @@ -930,6 +931,9 @@ class Role(Model): #: The short text the host's list carries, until `description` arrives. teaser: str = "" original_url: str = "" + #: Where a Jobindex ad's own page leads, found when the text is fetched + #: (`app.jobs.posting`); '' until then and for every other host. + posting_url: str = "" published_at: str #: The date this role counts as posted: see `posted_at`. Used for #: `recency` and the feed's newest order; `first_seen` stays what it @@ -1039,8 +1043,9 @@ class RoleRow(Model): there is one, else the description's opening, cut to `EXCERPT_LIMIT` characters. The pay fields are what the row's badge reads: the three normalised columns, the line the board stated, and `salary_state`, - derived as on `Role`. `original_url` is one short string, kept so the - `o` key opens a selected row's posting without opening the reader. + derived as on `Role`. `original_url` and `posting_url` are short strings, + kept so the `o` key opens a selected row's posting without opening the + reader. `seniority` is the title's level until extract reads the posting, '' when neither says. `copies` counts the open copies the feed hides behind this row.""" @@ -1059,6 +1064,7 @@ class RoleRow(Model): currency: str = "" salary_text: str = "" original_url: str = "" + posting_url: str = "" posted_at: str = "" score: float | None = None gated_reason: str = "" diff --git a/app/static/roles.js b/app/static/roles.js index 7814ac4..8c597a5 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -903,16 +903,23 @@ return n > 1 ? `×${n}` : ""; } + /* Where a role's posting is: the ad's own page once it is known + (`posting_url`, found for a Jobindex role when its text is fetched), else + the link the board listed. Either way only an http(s) link. */ + function linkOf(x) { + return safeUrl(x?.posting_url) || safeUrl(x?.original_url); + } + /* The other postings of this role, each a link to where it is posted, labelled by host and place, as newsdesk's "Also covered by". */ function alsoPosted(r) { const copies = (Array.isArray(r.copies) ? r.copies : []) - .filter((c) => safeUrl(c.original_url)); + .filter((c) => linkOf(c)); if (!copies.length) return ""; return `

Also posted

${copies.map((c) => { - const url = safeUrl(c.original_url); + const url = linkOf(c); const when = c.posted_at ? ago(c.posted_at) : ""; return ` ${esc(c.host || host(url))} @@ -925,7 +932,7 @@ link opens: the copies' own links, and the role's link for its own. */ function placeEntries(r) { const seen = new Set(); - const own = safeUrl(r.original_url); + const own = linkOf(r); const out = []; for (const p of Array.isArray(r.places) ? r.places : []) { const url = p.url ? safeUrl(p.url) : own; @@ -1312,7 +1319,7 @@ el.dataset.uid = state.readingUid; if (r.title) document.title = `${r.title} — roledesk`; - const out = safeUrl(r.original_url); + const out = linkOf(r); const text = (r.article_text || r.description || "").trim(); const body = error ? `

This role could not be loaded: ${esc(error)}

` @@ -1502,7 +1509,7 @@ const uid = state.readingUid || state.items[state.selected]?.uid; const role = state.reading?.uid === uid ? state.reading : state.items.find((r) => r.uid === uid); - const url = safeUrl(role?.original_url); + const url = linkOf(role); if (!url) return; window.open(url, "_blank", "noopener"); markRead(uid); diff --git a/app/storage/schema.py b/app/storage/schema.py index 7fe1676..eb3fa72 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -69,6 +69,9 @@ CREATE TABLE IF NOT EXISTS roles ( description TEXT NOT NULL DEFAULT '', -- the posting's body text feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over original_url TEXT NOT NULL DEFAULT '', -- the posting page + -- Where a search host's own page leads: Jobindex's "Se jobbet" link, found + -- when the posting text is fetched (app.jobs.posting). Never written by an upsert. + posting_url TEXT NOT NULL DEFAULT '', published_at TEXT NOT NULL, -- what the source claims, UTC -- What storing postings buys. first_seen is when the app itself first -- stored the role; posted_at (app.models.posted_at) is the earlier of @@ -623,6 +626,9 @@ UPDATE roles SET seniority_model = '' UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''), NULLIF(seniority_stated, ''), seniority_title); """), + # A Python step: it adds the column only where it is missing. + (26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is " + "fetched", lambda conn: _add_posting_url(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -864,6 +870,14 @@ def _add_fetch_terms(conn: sqlite3.Connection) -> None: conn.execute("ALTER TABLE agents ADD COLUMN fetch_terms TEXT NOT NULL DEFAULT '[]'") +def _add_posting_url(conn: sqlite3.Connection) -> None: + """Migration 26's step: `roles.posting_url`, empty for every role. Runs + twice without harm.""" + columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")} + if "posting_url" not in columns: + conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''") + + def _add_all_roles(conn: sqlite3.Connection) -> None: """Migration 21's step. The column; the tag for every company that is a favourite of any agent, since favourite no longer widens what a pull diff --git a/app/storage/store.py b/app/storage/store.py index 1e15076..9ecd17d 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -803,7 +803,7 @@ class Store: """The open copies of one kept role, oldest first; empty for a copy or a role without any. See `Role.copies`.""" return [RoleCopy.from_row(r) for r in self._rows( - f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles + f"""SELECT uid, {_HOST} AS host, place, original_url, posting_url, posted_at FROM roles WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")} ORDER BY posted_at, uid""", (uid, uid))] @@ -1060,7 +1060,7 @@ class Store: "SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, " "roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, " "roles.salary_max, roles.currency, roles.salary_text, roles.original_url, " - "roles.posted_at, roles.read_at, " + "roles.posting_url, roles.posted_at, roles.read_at, " "v.vote, v.voted_at, " f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt") order = "roles.posted_at DESC, roles.uid" @@ -1629,6 +1629,12 @@ class Store: return None return self.role(uid) + def set_posting_url(self, uid: str, url: str) -> bool: + """Keep where a search host's own page leads. False for an unknown + uid. Only this writes it: `upsert_role` leaves it alone, so a pull + keeps it.""" + return bool(self._write("UPDATE roles SET posting_url = ? WHERE uid = ?", (url, uid))) + # --------------------------------------------------------------- enrich def enrich_waiting(self, top: int, floor: float, diff --git a/plugins/readers/jobindex.py b/plugins/readers/jobindex.py index c7bb7ce..af4f881 100644 --- a/plugins/readers/jobindex.py +++ b/plugins/readers/jobindex.py @@ -15,6 +15,13 @@ the paragraphs are an excerpt of the ad, handed over as `teaser`. A crawled ad ("robot job") has no logo, so its company is what follows the title's last comma, or '' when there is none. An answer with no `` is not a search result and raises, since an empty list would read as a search that ran out. + +The `` opens Jobindex's `vis-job` page, and `posting_link` reads the +"Se jobbet" button on it: the employer's site, The Hub or Jobindex's own +`jobannonce`, checked on 2026-09-30. It is not part of the search: fetching +that page counts as a click in Jobindex's statistics, so `app.jobs.posting` +does it only for a role somebody opened. The page's "other ads" cards carry +the same words on `/c?t=` redirects, which are not this ad and are skipped. """ from __future__ import annotations @@ -24,7 +31,7 @@ import re import xml.etree.ElementTree as ET from collections.abc import Callable from typing import Any -from urllib.parse import urlencode +from urllib.parse import urlencode, urljoin, urlsplit from desk.sources import strip_html @@ -62,6 +69,8 @@ _AREA = re.compile(r'class="jix_robotjob--area">(.*?)', re.IGNORECASE | r _PARAGRAPHS = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) #: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden". _SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende") +_SE_JOBBET = re.compile(r"]*)>\s*Se jobbet\s*
", re.IGNORECASE) +_HREF = re.compile(r'\bhref\s*=\s*"([^"]*)"', re.IGNORECASE) def allowed(slug: str) -> Callable[[str], bool]: @@ -77,6 +86,20 @@ def clean_params(params: dict[str, Any]) -> dict[str, Any]: return out +def posting_link(page_html: str) -> str: + """The "Se jobbet" link of a `vis-job` page: where the ad itself is, as an + absolute URL, or '' when the page has none. A `/c?t=` link is another ad's + redirect and is skipped.""" + for tag in _SE_JOBBET.finditer(page_html): + found = _HREF.search(tag.group(1)) + if not found: + continue + url = urljoin("https://www.jobindex.dk/", html.unescape(found.group(1)).strip()) + if is_web_url(url) and urlsplit(url).path != "/c": + return url + return "" + + def _text(fragment: str) -> str: """HTML to plain text. Jobindex breaks titles after a slash with a zero-width space, which would otherwise end up in `title_key`.""" diff --git a/tests/fixtures/jobindex/vis-job-employer.html b/tests/fixtures/jobindex/vis-job-employer.html new file mode 100644 index 0000000..392e8ef --- /dev/null +++ b/tests/fixtures/jobindex/vis-job-employer.html @@ -0,0 +1,32 @@ + + + + Løsningsarkitekt til Købstædernes Forsikring - 1702752 | Jobindex + + + +
+ +
+
+ + + + + + Gem job + + + + + Se jobbet + + +
+ Se jobbet + +
+ + + + diff --git a/tests/fixtures/jobindex/vis-job-jobannonce.html b/tests/fixtures/jobindex/vis-job-jobannonce.html new file mode 100644 index 0000000..f34707d --- /dev/null +++ b/tests/fixtures/jobindex/vis-job-jobannonce.html @@ -0,0 +1,114 @@ + + + + Lead Software Engineer - Research Tech Delivery, AI & Digital Innovation - 14009457 | Jobindex + + + + + +
+
+ + + + + + Gem job + + + + + Se jobbet + + +
+ Se jobbet + +
+ + + + + + + + + + + + + + + + +
+ +
+
+ + + + + + Gem job + + + + + Se jobbet + + +
+ Se jobbet + +
+ + + + + + + + + + + + +
+ + +
+