Merge the Jobindex link-out: the Se jobbet link kept as posting_url (wp-d-jobindex)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:45:27 +02:00
commit 27e24d0342
19 files changed
+695 -29

No files matched your search

+5 -4
View File
@@ -13,6 +13,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off
discover.py which ATS a company's careers site runs on, from its pages
posting.py one role's posting page fetched as text, on demand
reach.py which countries a pull asks each big board for, from the agents
"""
@@ -20,8 +21,8 @@ from __future__ import annotations
from typing import Any
from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages,
summarise)
from app.jobs import (cascade, discover, embed, extract, posting, pull, reach, score, sources,
stages, summarise)
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -40,5 +41,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
return config, Store(config.database)
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
"reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "posting",
"pull", "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
+136
View File
@@ -0,0 +1,136 @@
"""A role's posting page, fetched once as plain text on demand.
`fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's
link, keep the text as `article_text`. It is on demand, never in the pull,
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
and one click for a posting somebody opened is fair.
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
ad and a "Se jobbet" button to the posting itself: the employer's site, The
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
`posting_url` and scraped for the text; the `vis-job` page's own text is the
fallback when there is no link or it gives nothing. A stored `posting_url` is
followed directly the next time, so the `vis-job` page is not fetched twice
for the same text.
Everything that stops the fetch is a `PostingError` with the HTTP status and
message the route answers with.
"""
from __future__ import annotations
from app.models import Role
from app.storage import Store
from desk.tools import scrape
from desk.tools.checks import Page
from plugins.readers import jobindex
#: What the read page toasts when a posting page gave nothing readable, by
#: `desk.tools.failures` category.
UNREADABLE = {
"timeout": "The posting page took too long to answer. Try again in a minute.",
"unreachable": "The posting page could not be reached.",
"rate_limited": "The posting site is refusing requests for now. Try again later.",
"server_error": "The posting site answered with an error. Try again later.",
"not_found": "The posting page is gone, so the role has probably closed.",
"denied": "The posting site refused to show the page.",
"consent_wall": "The posting page showed a consent banner instead of the posting.",
"js_required": "The posting page only renders in a browser. Open it with the link instead.",
"empty": "The posting page had no readable text on it.",
}
NOTHING_READABLE = "Nothing readable came back from the posting page."
class PostingError(Exception):
"""A posting text that was not fetched. `status` is the HTTP status the
route answers with, `str(error)` the message."""
status = 422
class UnknownRole(PostingError):
status = 404
class AlreadyFetched(PostingError):
status = 409
class NoLink(PostingError):
"""The role has no link to fetch the posting from."""
class Unreadable(PostingError):
"""The page answered with nothing readable; `failure` is its
`desk.tools.failures` category, '' when none was named."""
def __init__(self, failure: str = "") -> None:
super().__init__(UNREADABLE.get(failure, NOTHING_READABLE))
self.failure = failure
def _plain(got: scrape.Scraped) -> str:
return got.content.plain_text.strip() if got.content is not None else ""
def _keeping_html(url: str) -> tuple[scrape.Scraped, str]:
"""`scrape.scrape` over one request, and the HTML that request answered
with, '' when none did."""
pages: list[str] = []
def fetch(target: str) -> Page | None:
page = scrape.HTTP.fetch(target)
if page is not None:
pages.append(page.html)
return page
got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)})
return got, pages[0] if pages else ""
def _is_jobindex(role: Role) -> bool:
return role.raw.get("provider") == "jobindex"
def _jobindex_text(store: Store, role: Role) -> str:
"""The text of a Jobindex role: the page its "Se jobbet" button leads to,
else the `vis-job` page's own. Raises `Unreadable` when neither reads."""
own: scrape.Scraped | None = None
link, failure = role.posting_url, ""
if not link:
own, html = _keeping_html(role.original_url)
link = jobindex.posting_link(html)
if link:
store.set_posting_url(role.uid, link)
if link and link != role.original_url:
got = scrape.scrape(link)
if text := _plain(got):
return text
failure = got.failure
own = own or scrape.scrape(role.original_url)
if text := _plain(own):
return text
raise Unreadable(failure or own.failure)
def fetch_posting(store: Store, uid: str) -> Role:
"""Fetch the posting page once and keep it as plain text, for a role
whose description is a teaser. Synchronous: one request the person asked
for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`."""
role = store.role(uid)
if role is None:
raise UnknownRole(f"no such role: {uid}")
if role.article_text:
raise AlreadyFetched("The posting text is already fetched.")
if not role.original_url:
raise NoLink("This role has no link to fetch the posting from.")
if _is_jobindex(role):
text = _jobindex_text(store, role)
else:
got = scrape.scrape(role.original_url)
if not (text := _plain(got)):
raise Unreadable(got.failure)
updated = store.set_article_text(uid, text)
if updated is None:
raise UnknownRole(f"no such role: {uid}")
return updated
+8 -2
View File
@@ -874,6 +874,7 @@ class RoleCopy(Model):
host: str = ""
place: str = ""
original_url: str = ""
posting_url: str = ""
posted_at: str = ""
@@ -930,6 +931,9 @@ class Role(Model):
#: The short text the host's list carries, until `description` arrives.
teaser: str = ""
original_url: str = ""
#: Where a Jobindex ad's own page leads, found when the text is fetched
#: (`app.jobs.posting`); '' until then and for every other host.
posting_url: str = ""
published_at: str
#: The date this role counts as posted: see `posted_at`. Used for
#: `recency` and the feed's newest order; `first_seen` stays what it
@@ -1039,8 +1043,9 @@ class RoleRow(Model):
there is one, else the description's opening, cut to `EXCERPT_LIMIT`
characters. The pay fields are what the row's badge reads: the three
normalised columns, the line the board stated, and `salary_state`,
derived as on `Role`. `original_url` is one short string, kept so the
`o` key opens a selected row's posting without opening the reader.
derived as on `Role`. `original_url` and `posting_url` are short strings,
kept so the `o` key opens a selected row's posting without opening the
reader.
`seniority` is the title's level until extract reads the posting, ''
when neither says. `copies` counts the open copies the feed hides behind
this row."""
@@ -1059,6 +1064,7 @@ class RoleRow(Model):
currency: str = ""
salary_text: str = ""
original_url: str = ""
posting_url: str = ""
posted_at: str = ""
score: float | None = None
gated_reason: str = ""
+12 -5
View File
@@ -903,16 +903,23 @@
return n > 1 ? `<span class="dupes" title="Posted ${n} times">×${n}</span>` : "";
}
/* Where a role's posting is: the ad's own page once it is known
(`posting_url`, found for a Jobindex role when its text is fetched), else
the link the board listed. Either way only an http(s) link. */
function linkOf(x) {
return safeUrl(x?.posting_url) || safeUrl(x?.original_url);
}
/* The other postings of this role, each a link to where it is posted,
labelled by host and place, as newsdesk's "Also covered by". */
function alsoPosted(r) {
const copies = (Array.isArray(r.copies) ? r.copies : [])
.filter((c) => safeUrl(c.original_url));
.filter((c) => linkOf(c));
if (!copies.length) return "";
return `<div class="also">
<h2 class="reader-label">Also posted</h2>
${copies.map((c) => {
const url = safeUrl(c.original_url);
const url = linkOf(c);
const when = c.posted_at ? ago(c.posted_at) : "";
return `<a href="${esc(url)}" target="_blank" rel="noopener">
<b>${esc(c.host || host(url))}</b>
@@ -925,7 +932,7 @@
link opens: the copies' own links, and the role's link for its own. */
function placeEntries(r) {
const seen = new Set();
const own = safeUrl(r.original_url);
const own = linkOf(r);
const out = [];
for (const p of Array.isArray(r.places) ? r.places : []) {
const url = p.url ? safeUrl(p.url) : own;
@@ -1312,7 +1319,7 @@
el.dataset.uid = state.readingUid;
if (r.title) document.title = `${r.title} — roledesk`;
const out = safeUrl(r.original_url);
const out = linkOf(r);
const text = (r.article_text || r.description || "").trim();
const body = error
? `<p class="reader-note">This role could not be loaded: ${esc(error)}</p>`
@@ -1502,7 +1509,7 @@
const uid = state.readingUid || state.items[state.selected]?.uid;
const role = state.reading?.uid === uid ? state.reading
: state.items.find((r) => r.uid === uid);
const url = safeUrl(role?.original_url);
const url = linkOf(role);
if (!url) return;
window.open(url, "_blank", "noopener");
markRead(uid);
+14
View File
@@ -69,6 +69,9 @@ CREATE TABLE IF NOT EXISTS roles (
description TEXT NOT NULL DEFAULT '', -- the posting's body text
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
original_url TEXT NOT NULL DEFAULT '', -- the posting page
-- Where a search host's own page leads: Jobindex's "Se jobbet" link, found
-- when the posting text is fetched (app.jobs.posting). Never written by an upsert.
posting_url TEXT NOT NULL DEFAULT '',
published_at TEXT NOT NULL, -- what the source claims, UTC
-- What storing postings buys. first_seen is when the app itself first
-- stored the role; posted_at (app.models.posted_at) is the earlier of
@@ -623,6 +626,9 @@ UPDATE roles SET seniority_model = ''
UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
NULLIF(seniority_stated, ''), seniority_title);
"""),
# A Python step: it adds the column only where it is missing.
(26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is "
"fetched", lambda conn: _add_posting_url(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -864,6 +870,14 @@ def _add_fetch_terms(conn: sqlite3.Connection) -> None:
conn.execute("ALTER TABLE agents ADD COLUMN fetch_terms TEXT NOT NULL DEFAULT '[]'")
def _add_posting_url(conn: sqlite3.Connection) -> None:
"""Migration 26's step: `roles.posting_url`, empty for every role. Runs
twice without harm."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
if "posting_url" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
def _add_all_roles(conn: sqlite3.Connection) -> None:
"""Migration 21's step. The column; the tag for every company that is a
favourite of any agent, since favourite no longer widens what a pull
+8 -2
View File
@@ -803,7 +803,7 @@ class Store:
"""The open copies of one kept role, oldest first; empty for a copy
or a role without any. See `Role.copies`."""
return [RoleCopy.from_row(r) for r in self._rows(
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
f"""SELECT uid, {_HOST} AS host, place, original_url, posting_url, posted_at FROM roles
WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")}
ORDER BY posted_at, uid""", (uid, uid))]
@@ -1060,7 +1060,7 @@ class Store:
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
"roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, "
"roles.salary_max, roles.currency, roles.salary_text, roles.original_url, "
"roles.posted_at, roles.read_at, "
"roles.posting_url, roles.posted_at, roles.read_at, "
"v.vote, v.voted_at, "
f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt")
order = "roles.posted_at DESC, roles.uid"
@@ -1629,6 +1629,12 @@ class Store:
return None
return self.role(uid)
def set_posting_url(self, uid: str, url: str) -> bool:
"""Keep where a search host's own page leads. False for an unknown
uid. Only this writes it: `upsert_role` leaves it alone, so a pull
keeps it."""
return bool(self._write("UPDATE roles SET posting_url = ? WHERE uid = ?", (url, uid)))
# --------------------------------------------------------------- enrich
def enrich_waiting(self, top: int, floor: float,
+24 -1
View File
@@ -15,6 +15,13 @@ the paragraphs are an excerpt of the ad, handed over as `teaser`. A crawled ad
("robot job") has no logo, so its company is what follows the title's last
comma, or '' when there is none. An answer with no `<channel>` is not a search
result and raises, since an empty list would read as a search that ran out.
The `<link>` opens Jobindex's `vis-job` page, and `posting_link` reads the
"Se jobbet" button on it: the employer's site, The Hub or Jobindex's own
`jobannonce`, checked on 2026-09-30. It is not part of the search: fetching
that page counts as a click in Jobindex's statistics, so `app.jobs.posting`
does it only for a role somebody opened. The page's "other ads" cards carry
the same words on `/c?t=` redirects, which are not this ad and are skipped.
"""
from __future__ import annotations
@@ -24,7 +31,7 @@ import re
import xml.etree.ElementTree as ET
from collections.abc import Callable
from typing import Any
from urllib.parse import urlencode
from urllib.parse import urlencode, urljoin, urlsplit
from desk.sources import strip_html
@@ -62,6 +69,8 @@ _AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | r
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
_SE_JOBBET = re.compile(r"<a\b([^>]*)>\s*Se jobbet\s*</a>", re.IGNORECASE)
_HREF = re.compile(r'\bhref\s*=\s*"([^"]*)"', re.IGNORECASE)
def allowed(slug: str) -> Callable[[str], bool]:
@@ -77,6 +86,20 @@ def clean_params(params: dict[str, Any]) -> dict[str, Any]:
return out
def posting_link(page_html: str) -> str:
"""The "Se jobbet" link of a `vis-job` page: where the ad itself is, as an
absolute URL, or '' when the page has none. A `/c?t=` link is another ad's
redirect and is skipped."""
for tag in _SE_JOBBET.finditer(page_html):
found = _HREF.search(tag.group(1))
if not found:
continue
url = urljoin("https://www.jobindex.dk/", html.unescape(found.group(1)).strip())
if is_web_url(url) and urlsplit(url).path != "/c":
return url
return ""
def _text(fragment: str) -> str:
"""HTML to plain text. Jobindex breaks titles after a slash with a
zero-width space, which would otherwise end up in `title_key`."""
+32
View File
@@ -0,0 +1,32 @@
<!DOCTYPE html>
<html lang="da-DK">
<head>
<title>Løsningsarkitekt til Købstædernes Forsikring - 1702752 | Jobindex</title>
</head>
<body>
<!-- Saved from a vis-job page on 2026-09-30, cut to the "Se jobbet" buttons and what follows them. -->
</div>
</div>
<div class="mt-4">
<div class="jix-toolbar__buttons">
<span class="btn-list__element">
<span class="jobad-element-myjob" data-state="unsaved" data-tid="h1702752">
<a class="btn btn-secondary btn-sm" href="/bruger/dine-job/h1702752/gem">
<svg class="svg-icon svg-icon--mini m-0" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-button-myjob-unsaved"></use></svg>
<span class="ml-1">Gem job</span>
</a></span>
</span>
<span class="btn-list__element d-none d-md-inline">
<a class="btn btn-sm btn-primary seejobdesktop" data-click="/c?t=h1702752&amp;ctx=w" href="https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring?promotion=2220670-jobindex" rel="noopener" target="_blank">Se jobbet</a>
</span>
</div>
<a class="btn btn-sm btn-block btn-primary d-md-none mt-2 seejobmobil" data-click="/c?t=h1702752&amp;ctx=w" href="https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring?promotion=2220670-jobindex" rel="noopener" target="_blank">Se jobbet</a>
</div>
</body>
</html>
+114
View File
@@ -0,0 +1,114 @@
<!DOCTYPE html>
<html lang="da-DK">
<head>
<title>Lead Software Engineer - Research Tech Delivery, AI &amp; Digital Innovation - 14009457 | Jobindex</title>
</head>
<body>
<!-- Saved from a vis-job page on 2026-09-30, cut to the "Se jobbet" buttons and what follows them. -->
</div>
</div>
<div class="mt-4">
<div class="jix-toolbar__buttons">
<span class="btn-list__element">
<span class="jobad-element-myjob" data-state="unsaved" data-tid="r14009457">
<a class="btn btn-secondary btn-sm" href="/bruger/dine-job/r14009457/gem">
<svg class="svg-icon svg-icon--mini m-0" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-button-myjob-unsaved"></use></svg>
<span class="ml-1">Gem job</span>
</a></span>
</span>
<span class="btn-list__element d-none d-md-inline">
<a class="btn btn-sm btn-primary seejobdesktop" data-click="/c?t=r14009457&amp;ctx=w" href="https://www.jobindex.dk/jobannonce/r14009457/lead-software-engineer-research-tech-delivery-ai-digital-innovation" rel="noopener" target="_blank">Se jobbet</a>
</span>
</div>
<a class="btn btn-sm btn-block btn-primary d-md-none mt-2 seejobmobil" data-click="/c?t=r14009457&amp;ctx=w" href="https://www.jobindex.dk/jobannonce/r14009457/lead-software-engineer-research-tech-delivery-ai-digital-innovation" rel="noopener" target="_blank">Se jobbet</a>
</div>
</span>
</div>
</div>
</div>
</div>
</div>
<aside class="offset-lg-1 col-md-3">
<div class="vp-card mb-4">
<div class="vp-card__follow">
<span class="vp-card__follow-count">22.134 følgere</span>
<a href="/bruger/dine-virksomheder/258" id="widgets_follow_company_38e66c4c-e28a-46b6-825c-91496291af00" class="jix-follow-company" role="button" data-jix-login="user" data-jix-login-redirect="/bruger/dine-virksomheder/258"><span class="jix-follow-text-active">Følger</span><span class="jix-follow-text-inactive">Følg</span></a>
</div>
<a class="vp-card__name" href="/virksomhed/258/novo#om-virksomhed">Novo</a>
<div class="vp-card__data">
<a class="vp-card__ratings" href="/virksomhed/258/novo#evalueringer">
<div class="vp-card__rating-stars" title="5 ud af 5 stjerner"><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg></div>
<div class="vp-card__ratings-count">2671 evalueringer</div>
</a> <div class="jix-link-block"><a href="/jobsoegning/virksomhed/258/novo"><span>67 jobannoncer</span><span aria-hidden="true"><svg class="svg-icon svg-icon--sm"><use xlink:href="#svg-linkarrow-right" xmlns:xlink="http://www.w3.org/1999/xlink"></use></svg></span></a></div>
</div>
<a class="btn btn-secondary btn-sm btn-block" href="/virksomhed/258/novo#om-virksomhed">Om virksomheden</a>
</div>
</aside>
</div>
</section>
<section class="bg-surface-1 mt-4">
<div class="recommended-jobs-container" id="recommended-jobs">
<h2 class="mb-4">Anbefalede job</h2>
<div class="job-card-container job-card-container--stacked">
<article class="job-card" data-tid="h1700212"
data-recommender-source="showjob"
data-recommender-origin-tid="r14009457"
>
<div class="job-card__pubdate">
Indrykket:
<time datetime="2026-09-21">21. september 2026</time>
</div>
<a class="job-card__headline" href="/c?t=h1700212&amp;ctx=w&amp;ctt=r&amp;u=22713329">
Senior Embedded Engineer / Software Developer
</a> <p class="job-card__company">Welltec A/S</p>
<div class="job-card__footer">
<div class="job-card__area">
<svg class="svg-icon svg-icon--sm"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-geolocation"></use></svg>
<span class="job-card__area-text">Allerød</span>
</div>
<a class="btn btn-secondary btn-sm btn-block" href="/c?t=h1700212&amp;ctx=w&amp;ctt=r&amp;u=22713329">Se jobbet</a>
</div>
</article>
<article class="job-card" data-tid="h1695228"
data-recommender-source="showjob"
data-recommender-origin-tid="r14009457"
>
<div class="job-card__pubdate">
Indrykket:
<time datetime="2026-09-03">3. september 2026</time>
</div>
<a class="job-card__headline" href="/c?t=h1695228&amp;ctx=w&amp;ctt=r&amp;u=22687519">
Bliv vores nye Lead Software Engineer og vær med til at udvikle Nykredit-kon­cer­nens re­al­kre­dit­løs­nin­ger
</a> <p class="job-card__company">Nykredit</p>
<div class="job-card__footer">
<div class="job-card__area">
<svg class="svg-icon svg-icon--sm"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-geolocation"></use></svg>
<span class="job-card__area-text">Nordhavn</span>
</div>
<a class="btn btn-secondary btn-sm btn-block" href="/c?t=h1695228&amp;ctx=w&amp;ctt=r&amp;u=22687519">Se jobbet</a>
</div>
</article>
</body>
</html>
+94
View File
@@ -0,0 +1,94 @@
<!DOCTYPE html>
<html lang="da-DK">
<head>
<title>Senior Backend Developer - 14013846 | Jobindex</title>
</head>
<body>
<!-- Saved from a vis-job page on 2026-09-30, cut to the "Se jobbet" buttons and what follows them. -->
</div>
</div>
<div class="mt-4">
<div class="jix-toolbar__buttons">
<span class="btn-list__element">
<span class="jobad-element-myjob" data-state="unsaved" data-tid="r14013846">
<a class="btn btn-secondary btn-sm" href="/bruger/dine-job/r14013846/gem">
<svg class="svg-icon svg-icon--mini m-0" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-button-myjob-unsaved"></use></svg>
<span class="ml-1">Gem job</span>
</a></span>
</span>
<span class="btn-list__element d-none d-md-inline">
<a class="btn btn-sm btn-primary seejobdesktop" data-click="/c?t=r14013846&amp;ctx=w" href="https://thehub.io/jobs/6ab5030dfd5518f38b6960b2?utm_source=jobindex&amp;utm_medium=jobboard&amp;utm_content=dk" rel="noopener" target="_blank">Se jobbet</a>
</span>
</div>
<a class="btn btn-sm btn-block btn-primary d-md-none mt-2 seejobmobil" data-click="/c?t=r14013846&amp;ctx=w" href="https://thehub.io/jobs/6ab5030dfd5518f38b6960b2?utm_source=jobindex&amp;utm_medium=jobboard&amp;utm_content=dk" rel="noopener" target="_blank">Se jobbet</a>
</div>
</span>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
<section class="bg-surface-1 mt-4">
<div class="recommended-jobs-container" id="recommended-jobs">
<h2 class="mb-4">Anbefalede job</h2>
<div class="job-card-container job-card-container--stacked">
<article class="job-card" data-tid="h1700172"
data-recommender-source="showjob"
data-recommender-origin-tid="r14013846"
>
<div class="job-card__pubdate">
Indrykket:
<time datetime="2026-09-21">21. september 2026</time>
</div>
<a class="job-card__headline" href="/c?t=h1700172&amp;ctx=w&amp;ctt=r&amp;u=22713244">
Senior Developer
</a> <p class="job-card__company">Gubra A/S</p>
<div class="job-card__footer">
<div class="job-card__area">
<svg class="svg-icon svg-icon--sm"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-geolocation"></use></svg>
<span class="job-card__area-text">Hørsholm</span>
</div>
<a class="btn btn-secondary btn-sm btn-block" href="/c?t=h1700172&amp;ctx=w&amp;ctt=r&amp;u=22713244">Se jobbet</a>
</div>
</article>
<article class="job-card" data-tid="h1696175"
data-recommender-source="showjob"
data-recommender-origin-tid="r14013846"
>
<div class="job-card__pubdate">
Indrykket:
<time datetime="2026-09-07">7. september 2026</time>
</div>
<a class="job-card__headline" href="/c?t=h1696175&amp;ctx=w&amp;ctt=r&amp;u=22692389">
Senior .NET Developer - Mi­cro­ser­vi­ces &amp; In­te­gra­tions
</a> <p class="job-card__company">twoday A/S</p>
<div class="job-card__footer">
<div class="job-card__area">
<svg class="svg-icon svg-icon--sm"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-geolocation"></use></svg>
<span class="job-card__area-text">Nordhavn</span>
</div>
<a class="btn btn-secondary btn-sm btn-block" href="/c?t=h1696175&amp;ctx=w&amp;ctt=r&amp;u=22692389">Se jobbet</a>
</div>
</article>
</body>
</html>
+2 -2
View File
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25]
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21, 22, 23, 24, 25]
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26]
# The tag of 21 is All countries and All roles since 24.
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
+1 -1
View File
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25]
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+1 -1
View File
@@ -437,7 +437,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
conn.execute("PRAGMA user_version = 22")
migrated = Store(path)
assert migrated.migrations_applied == [23, 24, 25]
assert migrated.migrations_applied == [23, 24, 25, 26]
assert migrated.agents()[0].fetch_terms == []
+1 -1
View File
@@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
conn.execute("PRAGMA user_version = 23")
migrated = Store(path)
assert migrated.migrations_applied == [24, 25]
assert migrated.migrations_applied == [24, 25, 26]
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
+166
View File
@@ -0,0 +1,166 @@
"""`fetch_posting`: a role's posting text on demand, and for a Jobindex role
the page its "Se jobbet" button leads to. No network: `scrape` is stubbed."""
from __future__ import annotations
from datetime import UTC, datetime
from pathlib import Path
import pytest
from app.jobs import posting
from app.storage import Store
from desk.models import Item
from desk.tools import scrape
from desk.tools.article import ArticleContent
from desk.tools.checks import Page
from prepped import upsert
ROOT = Path(__file__).resolve().parents[1]
VIS_JOB = "https://www.jobindex.dk/vis-job/h1702752"
LINK = ("https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring"
"?promotion=2220670-jobindex")
PAGE = (ROOT / "tests" / "fixtures" / "jobindex" / "vis-job-employer.html").read_text("utf-8")
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def role(store: Store, url: str = "https://boards.example/acme/1", source_id: str = "1",
**raw: object) -> str:
item = Item(source_id=source_id, location=url, title="Engineer", body="A teaser.",
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
item.source = "ats"
upsert(store, item)
return item.uid
def jobindex_role(store: Store) -> str:
return role(store, VIS_JOB, provider="jobindex")
def content(text: str, url: str = "") -> scrape.Scraped:
return scrape.Scraped(ArticleContent(format="plain", content=text, source_url=url,
method="test"), "", "http", ())
def failed(category: str) -> scrape.Scraped:
return scrape.Scraped(None, category, "http", ())
class Web:
"""Stands in for the network: `pages` maps a URL to what `scrape.HTTP`
answers with, `texts` a URL to what the article of that page reads as."""
def __init__(self, monkeypatch: pytest.MonkeyPatch, pages: dict[str, str],
texts: dict[str, str]) -> None:
self.asked: list[str] = []
self.pages, self.texts = pages, texts
monkeypatch.setattr(scrape, "HTTP", scrape.Getter("http", self.fetch))
monkeypatch.setattr(scrape, "STANDARD", {"http": scrape.HTTP})
monkeypatch.setattr(scrape, "scrape", self.scrape)
def fetch(self, url: str) -> Page | None:
self.asked.append(url)
html = self.pages.get(url)
return None if html is None else Page(status=200, html=html, url=url, headers={})
def scrape(self, url: str, getters: object = None, first: str = "http") -> scrape.Scraped:
if getters is not None: # the page is fetched by the getter it was handed
page = getters["http"].fetch(url) # type: ignore[index]
return content(self.texts[url], url) if page and url in self.texts else failed("empty")
self.asked.append(url)
return content(self.texts[url], url) if url in self.texts else failed("not_found")
def test_a_role_that_cannot_be_fetched_says_why(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
Web(monkeypatch, {}, {})
with pytest.raises(posting.UnknownRole) as unknown:
posting.fetch_posting(store, "nope")
assert (unknown.value.status, str(unknown.value)) == (404, "no such role: nope")
bare = role(store, url="")
with pytest.raises(posting.NoLink) as nolink:
posting.fetch_posting(store, bare)
assert nolink.value.status == 422
def test_an_ordinary_role_is_scraped_from_its_link_once(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
uid = role(store)
web = Web(monkeypatch, {}, {})
with pytest.raises(posting.Unreadable) as gone:
posting.fetch_posting(store, uid)
assert gone.value.status == 422 and "gone" in str(gone.value)
assert web.asked == ["https://boards.example/acme/1"]
web.texts["https://boards.example/acme/1"] = " The whole posting.\n"
got = posting.fetch_posting(store, uid)
assert got.article_text == "The whole posting." and got.article_at
assert got.posting_url == ""
with pytest.raises(posting.AlreadyFetched) as again:
posting.fetch_posting(store, uid)
assert again.value.status == 409
def test_an_unknown_failure_reads_as_nothing_readable(store: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
uid = role(store)
web = Web(monkeypatch, {}, {})
monkeypatch.setattr(web, "scrape", lambda url, *a, **k: failed("weird"))
monkeypatch.setattr(scrape, "scrape", web.scrape)
with pytest.raises(posting.Unreadable, match="Nothing readable"):
posting.fetch_posting(store, uid)
def test_a_jobindex_role_follows_its_se_jobbet_link_and_keeps_it(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad.", LINK: "The real posting."})
got = posting.fetch_posting(store, uid)
assert got.article_text == "The real posting."
assert got.posting_url == LINK and got.original_url == VIS_JOB
assert web.asked == [VIS_JOB, LINK]
stored = store.role(uid)
assert stored is not None and stored.posting_url == LINK
def test_a_jobindex_role_falls_back_to_the_ads_own_text(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
# The link gives nothing readable: the link is kept, the vis-job text stored.
uid = jobindex_role(store)
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad."})
got = posting.fetch_posting(store, uid)
assert got.article_text == "The Jobindex ad." and got.posting_url == LINK
assert web.asked == [VIS_JOB, LINK]
# No button on the page: the ad's own text, no link, one request.
other = role(store, "https://www.jobindex.dk/vis-job/r2", "2", provider="jobindex")
web = Web(monkeypatch, {"https://www.jobindex.dk/vis-job/r2": "<p>Ingen knap</p>"},
{"https://www.jobindex.dk/vis-job/r2": "Ad two."})
got = posting.fetch_posting(store, other)
assert got.article_text == "Ad two." and got.posting_url == ""
assert web.asked == ["https://www.jobindex.dk/vis-job/r2"]
def test_a_jobindex_role_whose_pages_read_as_nothing_is_unreadable_but_keeps_the_link(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
Web(monkeypatch, {VIS_JOB: PAGE}, {})
with pytest.raises(posting.Unreadable) as error:
posting.fetch_posting(store, uid)
assert error.value.failure == "not_found"
stored = store.role(uid)
assert stored is not None and stored.posting_url == LINK and stored.article_text == ""
def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
uid = jobindex_role(store)
store.set_posting_url(uid, LINK)
web = Web(monkeypatch, {}, {LINK: "The real posting."})
assert posting.fetch_posting(store, uid).article_text == "The real posting."
assert web.asked == [LINK]
+1 -1
View File
@@ -314,7 +314,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [22, 23, 24, 25]
assert migrated.migrations_applied == [22, 23, 24, 25, 26]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
+36
View File
@@ -100,3 +100,39 @@ def test_an_answer_without_a_channel_raises() -> None:
def test_saved_params_keep_the_query_and_a_real_area() -> None:
assert jobindex.clean_params({"q": "x", "area": "3"}) == {"q": "x", "area": 3}
assert jobindex.clean_params({"q": "x", "area": 0}) == {"q": "x"}
# ------------------------------------------------- the link out of a vis-job page
PAGES = ROOT / "tests" / "fixtures" / "jobindex"
@pytest.mark.parametrize("name, link", [
("vis-job-employer.html",
"https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring"
"?promotion=2220670-jobindex"),
("vis-job-thehub.html",
"https://thehub.io/jobs/6ab5030dfd5518f38b6960b2"
"?utm_source=jobindex&utm_medium=jobboard&utm_content=dk"),
("vis-job-jobannonce.html",
"https://www.jobindex.dk/jobannonce/r14009457/"
"lead-software-engineer-research-tech-delivery-ai-digital-innovation"),
])
def test_posting_link_is_the_se_jobbet_button_of_a_saved_page(name: str, link: str) -> None:
assert jobindex.posting_link((PAGES / name).read_text(encoding="utf-8")) == link
def test_posting_link_skips_other_ads_and_is_empty_without_a_button() -> None:
other = '<a class="btn" href="/c?t=h1&amp;ctx=w&amp;ctt=r&amp;u=2">Se jobbet</a>'
assert jobindex.posting_link(other) == ""
assert jobindex.posting_link("<p>Ingen knap</p>") == ""
assert jobindex.posting_link("") == ""
mine = '<A HREF="https://a.example/x?a=1&amp;b=2" rel="noopener">\n Se jobbet </A>'
assert jobindex.posting_link(other + mine) == "https://a.example/x?a=1&b=2"
def test_posting_link_is_absolute_and_never_a_script() -> None:
assert jobindex.posting_link('<a href="/jobannonce/r1/x">Se jobbet</a>') \
== "https://www.jobindex.dk/jobannonce/r1/x"
assert jobindex.posting_link('<a href="javascript:alert(1)">Se jobbet</a>') == ""
assert jobindex.posting_link('<a class="x">Se jobbet</a>') == ""
+37 -7
View File
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 25
assert LATEST == 26
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -688,7 +688,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25]
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -730,7 +730,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
migrated = Store(path)
assert migrated.migrations_applied == [25]
assert migrated.migrations_applied == [25, 26]
with sqlite3.connect(path) as conn:
found = {sid: rest for sid, *rest in conn.execute(
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
@@ -738,6 +738,36 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
assert found == {sid: list(after) for sid, (_, after) in cases.items()}
def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path) -> None:
path = tmp_path / "v25.db"
store = Store(path)
upsert(store, posting("a"))
with sqlite3.connect(path) as conn:
conn.execute("ALTER TABLE roles DROP COLUMN posting_url")
conn.execute("PRAGMA user_version = 25")
migrated = Store(path)
assert migrated.migrations_applied == [26]
role = migrated.role(posting("a").uid)
assert role is not None and role.posting_url == ""
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 25") # the column is there already
assert Store(path).migrations_applied == [26]
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
upsert(store, posting("a"))
uid = posting("a").uid
assert store.set_posting_url(uid, "https://careers.example/jobs/1")
assert not store.set_posting_url("nope", "https://careers.example/jobs/1")
upsert(store, posting("a")) # the next pull reads the same role again
role = store.role(uid)
assert role is not None and role.posting_url == "https://careers.example/jobs/1"
row = next(r for r in store.roles_page() if r.uid == uid)
assert row.posting_url == "https://careers.example/jobs/1"
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
upsert(store, posting(place="Berlin", remote=True, salary_min="80000",
salary_max=95000, currency="EUR", department="Eng"))
+3 -2
View File
@@ -1224,7 +1224,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
assert set(row) == {
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
"seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state",
"original_url", "posted_at", "score", "gated_reason", "prompt", "vote",
"original_url", "posting_url", "posted_at", "score", "gated_reason", "prompt", "vote",
"voted_at", "read_at", "tags", "excerpt", "copies",
}
assert row["uid"] == uid
@@ -1734,7 +1734,8 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
assert kept["dup_of"] is None
assert kept["copies"] == [{
"uid": search.uid, "host": "jobindex", "place": "Ballerup, Denmark",
"original_url": search.location, "posted_at": kept["copies"][0]["posted_at"]}]
"original_url": search.location, "posting_url": "",
"posted_at": kept["copies"][0]["posted_at"]}]
copy = client.get(f"/api/roles/{search.uid}").json()
assert copy["dup_of"] == board.uid and copy["copies"] == []