Merge the Jobindex link-out: the Se jobbet link kept as posting_url (wp-d-jobindex)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
27e24d0342
19 files changed
+695
-29
No files matched your search
@@ -13,6 +13,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||
sources.py what a pull reads, changed by hand: boards and searches
|
||||
added, switched on or off
|
||||
discover.py which ATS a company's careers site runs on, from its pages
|
||||
posting.py one role's posting page fetched as text, on demand
|
||||
reach.py which countries a pull asks each big board for, from the agents
|
||||
"""
|
||||
|
||||
@@ -20,8 +21,8 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages,
|
||||
summarise)
|
||||
from app.jobs import (cascade, discover, embed, extract, posting, pull, reach, score, sources,
|
||||
stages, summarise)
|
||||
from app.jobs.pull import search_target, seed_searches
|
||||
from app.jobs.score import seed_agents
|
||||
|
||||
@@ -40,5 +41,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
|
||||
return config, Store(config.database)
|
||||
|
||||
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
|
||||
"reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "posting",
|
||||
"pull", "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""A role's posting page, fetched once as plain text on demand.
|
||||
|
||||
`fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's
|
||||
link, keep the text as `article_text`. It is on demand, never in the pull,
|
||||
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
|
||||
and one click for a posting somebody opened is fair.
|
||||
|
||||
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
|
||||
ad and a "Se jobbet" button to the posting itself: the employer's site, The
|
||||
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
|
||||
`posting_url` and scraped for the text; the `vis-job` page's own text is the
|
||||
fallback when there is no link or it gives nothing. A stored `posting_url` is
|
||||
followed directly the next time, so the `vis-job` page is not fetched twice
|
||||
for the same text.
|
||||
|
||||
Everything that stops the fetch is a `PostingError` with the HTTP status and
|
||||
message the route answers with.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.models import Role
|
||||
from app.storage import Store
|
||||
from desk.tools import scrape
|
||||
from desk.tools.checks import Page
|
||||
from plugins.readers import jobindex
|
||||
|
||||
#: What the read page toasts when a posting page gave nothing readable, by
|
||||
#: `desk.tools.failures` category.
|
||||
UNREADABLE = {
|
||||
"timeout": "The posting page took too long to answer. Try again in a minute.",
|
||||
"unreachable": "The posting page could not be reached.",
|
||||
"rate_limited": "The posting site is refusing requests for now. Try again later.",
|
||||
"server_error": "The posting site answered with an error. Try again later.",
|
||||
"not_found": "The posting page is gone, so the role has probably closed.",
|
||||
"denied": "The posting site refused to show the page.",
|
||||
"consent_wall": "The posting page showed a consent banner instead of the posting.",
|
||||
"js_required": "The posting page only renders in a browser. Open it with the link instead.",
|
||||
"empty": "The posting page had no readable text on it.",
|
||||
}
|
||||
NOTHING_READABLE = "Nothing readable came back from the posting page."
|
||||
|
||||
|
||||
class PostingError(Exception):
|
||||
"""A posting text that was not fetched. `status` is the HTTP status the
|
||||
route answers with, `str(error)` the message."""
|
||||
|
||||
status = 422
|
||||
|
||||
|
||||
class UnknownRole(PostingError):
|
||||
status = 404
|
||||
|
||||
|
||||
class AlreadyFetched(PostingError):
|
||||
status = 409
|
||||
|
||||
|
||||
class NoLink(PostingError):
|
||||
"""The role has no link to fetch the posting from."""
|
||||
|
||||
|
||||
class Unreadable(PostingError):
|
||||
"""The page answered with nothing readable; `failure` is its
|
||||
`desk.tools.failures` category, '' when none was named."""
|
||||
|
||||
def __init__(self, failure: str = "") -> None:
|
||||
super().__init__(UNREADABLE.get(failure, NOTHING_READABLE))
|
||||
self.failure = failure
|
||||
|
||||
|
||||
def _plain(got: scrape.Scraped) -> str:
|
||||
return got.content.plain_text.strip() if got.content is not None else ""
|
||||
|
||||
|
||||
def _keeping_html(url: str) -> tuple[scrape.Scraped, str]:
|
||||
"""`scrape.scrape` over one request, and the HTML that request answered
|
||||
with, '' when none did."""
|
||||
pages: list[str] = []
|
||||
|
||||
def fetch(target: str) -> Page | None:
|
||||
page = scrape.HTTP.fetch(target)
|
||||
if page is not None:
|
||||
pages.append(page.html)
|
||||
return page
|
||||
|
||||
got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)})
|
||||
return got, pages[0] if pages else ""
|
||||
|
||||
|
||||
def _is_jobindex(role: Role) -> bool:
|
||||
return role.raw.get("provider") == "jobindex"
|
||||
|
||||
|
||||
def _jobindex_text(store: Store, role: Role) -> str:
|
||||
"""The text of a Jobindex role: the page its "Se jobbet" button leads to,
|
||||
else the `vis-job` page's own. Raises `Unreadable` when neither reads."""
|
||||
own: scrape.Scraped | None = None
|
||||
link, failure = role.posting_url, ""
|
||||
if not link:
|
||||
own, html = _keeping_html(role.original_url)
|
||||
link = jobindex.posting_link(html)
|
||||
if link:
|
||||
store.set_posting_url(role.uid, link)
|
||||
if link and link != role.original_url:
|
||||
got = scrape.scrape(link)
|
||||
if text := _plain(got):
|
||||
return text
|
||||
failure = got.failure
|
||||
own = own or scrape.scrape(role.original_url)
|
||||
if text := _plain(own):
|
||||
return text
|
||||
raise Unreadable(failure or own.failure)
|
||||
|
||||
|
||||
def fetch_posting(store: Store, uid: str) -> Role:
|
||||
"""Fetch the posting page once and keep it as plain text, for a role
|
||||
whose description is a teaser. Synchronous: one request the person asked
|
||||
for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`."""
|
||||
role = store.role(uid)
|
||||
if role is None:
|
||||
raise UnknownRole(f"no such role: {uid}")
|
||||
if role.article_text:
|
||||
raise AlreadyFetched("The posting text is already fetched.")
|
||||
if not role.original_url:
|
||||
raise NoLink("This role has no link to fetch the posting from.")
|
||||
if _is_jobindex(role):
|
||||
text = _jobindex_text(store, role)
|
||||
else:
|
||||
got = scrape.scrape(role.original_url)
|
||||
if not (text := _plain(got)):
|
||||
raise Unreadable(got.failure)
|
||||
updated = store.set_article_text(uid, text)
|
||||
if updated is None:
|
||||
raise UnknownRole(f"no such role: {uid}")
|
||||
return updated
|
||||
+8
-2
@@ -874,6 +874,7 @@ class RoleCopy(Model):
|
||||
host: str = ""
|
||||
place: str = ""
|
||||
original_url: str = ""
|
||||
posting_url: str = ""
|
||||
posted_at: str = ""
|
||||
|
||||
|
||||
@@ -930,6 +931,9 @@ class Role(Model):
|
||||
#: The short text the host's list carries, until `description` arrives.
|
||||
teaser: str = ""
|
||||
original_url: str = ""
|
||||
#: Where a Jobindex ad's own page leads, found when the text is fetched
|
||||
#: (`app.jobs.posting`); '' until then and for every other host.
|
||||
posting_url: str = ""
|
||||
published_at: str
|
||||
#: The date this role counts as posted: see `posted_at`. Used for
|
||||
#: `recency` and the feed's newest order; `first_seen` stays what it
|
||||
@@ -1039,8 +1043,9 @@ class RoleRow(Model):
|
||||
there is one, else the description's opening, cut to `EXCERPT_LIMIT`
|
||||
characters. The pay fields are what the row's badge reads: the three
|
||||
normalised columns, the line the board stated, and `salary_state`,
|
||||
derived as on `Role`. `original_url` is one short string, kept so the
|
||||
`o` key opens a selected row's posting without opening the reader.
|
||||
derived as on `Role`. `original_url` and `posting_url` are short strings,
|
||||
kept so the `o` key opens a selected row's posting without opening the
|
||||
reader.
|
||||
`seniority` is the title's level until extract reads the posting, ''
|
||||
when neither says. `copies` counts the open copies the feed hides behind
|
||||
this row."""
|
||||
@@ -1059,6 +1064,7 @@ class RoleRow(Model):
|
||||
currency: str = ""
|
||||
salary_text: str = ""
|
||||
original_url: str = ""
|
||||
posting_url: str = ""
|
||||
posted_at: str = ""
|
||||
score: float | None = None
|
||||
gated_reason: str = ""
|
||||
|
||||
+12
-5
@@ -903,16 +903,23 @@
|
||||
return n > 1 ? `<span class="dupes" title="Posted ${n} times">×${n}</span>` : "";
|
||||
}
|
||||
|
||||
/* Where a role's posting is: the ad's own page once it is known
|
||||
(`posting_url`, found for a Jobindex role when its text is fetched), else
|
||||
the link the board listed. Either way only an http(s) link. */
|
||||
function linkOf(x) {
|
||||
return safeUrl(x?.posting_url) || safeUrl(x?.original_url);
|
||||
}
|
||||
|
||||
/* The other postings of this role, each a link to where it is posted,
|
||||
labelled by host and place, as newsdesk's "Also covered by". */
|
||||
function alsoPosted(r) {
|
||||
const copies = (Array.isArray(r.copies) ? r.copies : [])
|
||||
.filter((c) => safeUrl(c.original_url));
|
||||
.filter((c) => linkOf(c));
|
||||
if (!copies.length) return "";
|
||||
return `<div class="also">
|
||||
<h2 class="reader-label">Also posted</h2>
|
||||
${copies.map((c) => {
|
||||
const url = safeUrl(c.original_url);
|
||||
const url = linkOf(c);
|
||||
const when = c.posted_at ? ago(c.posted_at) : "";
|
||||
return `<a href="${esc(url)}" target="_blank" rel="noopener">
|
||||
<b>${esc(c.host || host(url))}</b>
|
||||
@@ -925,7 +932,7 @@
|
||||
link opens: the copies' own links, and the role's link for its own. */
|
||||
function placeEntries(r) {
|
||||
const seen = new Set();
|
||||
const own = safeUrl(r.original_url);
|
||||
const own = linkOf(r);
|
||||
const out = [];
|
||||
for (const p of Array.isArray(r.places) ? r.places : []) {
|
||||
const url = p.url ? safeUrl(p.url) : own;
|
||||
@@ -1312,7 +1319,7 @@
|
||||
el.dataset.uid = state.readingUid;
|
||||
if (r.title) document.title = `${r.title} — roledesk`;
|
||||
|
||||
const out = safeUrl(r.original_url);
|
||||
const out = linkOf(r);
|
||||
const text = (r.article_text || r.description || "").trim();
|
||||
const body = error
|
||||
? `<p class="reader-note">This role could not be loaded: ${esc(error)}</p>`
|
||||
@@ -1502,7 +1509,7 @@
|
||||
const uid = state.readingUid || state.items[state.selected]?.uid;
|
||||
const role = state.reading?.uid === uid ? state.reading
|
||||
: state.items.find((r) => r.uid === uid);
|
||||
const url = safeUrl(role?.original_url);
|
||||
const url = linkOf(role);
|
||||
if (!url) return;
|
||||
window.open(url, "_blank", "noopener");
|
||||
markRead(uid);
|
||||
|
||||
@@ -69,6 +69,9 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
description TEXT NOT NULL DEFAULT '', -- the posting's body text
|
||||
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
|
||||
original_url TEXT NOT NULL DEFAULT '', -- the posting page
|
||||
-- Where a search host's own page leads: Jobindex's "Se jobbet" link, found
|
||||
-- when the posting text is fetched (app.jobs.posting). Never written by an upsert.
|
||||
posting_url TEXT NOT NULL DEFAULT '',
|
||||
published_at TEXT NOT NULL, -- what the source claims, UTC
|
||||
-- What storing postings buys. first_seen is when the app itself first
|
||||
-- stored the role; posted_at (app.models.posted_at) is the earlier of
|
||||
@@ -623,6 +626,9 @@ UPDATE roles SET seniority_model = ''
|
||||
UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
NULLIF(seniority_stated, ''), seniority_title);
|
||||
"""),
|
||||
# A Python step: it adds the column only where it is missing.
|
||||
(26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is "
|
||||
"fetched", lambda conn: _add_posting_url(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -864,6 +870,14 @@ def _add_fetch_terms(conn: sqlite3.Connection) -> None:
|
||||
conn.execute("ALTER TABLE agents ADD COLUMN fetch_terms TEXT NOT NULL DEFAULT '[]'")
|
||||
|
||||
|
||||
def _add_posting_url(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 26's step: `roles.posting_url`, empty for every role. Runs
|
||||
twice without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
if "posting_url" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
||||
|
||||
|
||||
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 21's step. The column; the tag for every company that is a
|
||||
favourite of any agent, since favourite no longer widens what a pull
|
||||
|
||||
@@ -803,7 +803,7 @@ class Store:
|
||||
"""The open copies of one kept role, oldest first; empty for a copy
|
||||
or a role without any. See `Role.copies`."""
|
||||
return [RoleCopy.from_row(r) for r in self._rows(
|
||||
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
|
||||
f"""SELECT uid, {_HOST} AS host, place, original_url, posting_url, posted_at FROM roles
|
||||
WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")}
|
||||
ORDER BY posted_at, uid""", (uid, uid))]
|
||||
|
||||
@@ -1060,7 +1060,7 @@ class Store:
|
||||
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
|
||||
"roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, "
|
||||
"roles.salary_max, roles.currency, roles.salary_text, roles.original_url, "
|
||||
"roles.posted_at, roles.read_at, "
|
||||
"roles.posting_url, roles.posted_at, roles.read_at, "
|
||||
"v.vote, v.voted_at, "
|
||||
f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt")
|
||||
order = "roles.posted_at DESC, roles.uid"
|
||||
@@ -1629,6 +1629,12 @@ class Store:
|
||||
return None
|
||||
return self.role(uid)
|
||||
|
||||
def set_posting_url(self, uid: str, url: str) -> bool:
|
||||
"""Keep where a search host's own page leads. False for an unknown
|
||||
uid. Only this writes it: `upsert_role` leaves it alone, so a pull
|
||||
keeps it."""
|
||||
return bool(self._write("UPDATE roles SET posting_url = ? WHERE uid = ?", (url, uid)))
|
||||
|
||||
# --------------------------------------------------------------- enrich
|
||||
|
||||
def enrich_waiting(self, top: int, floor: float,
|
||||
|
||||
@@ -15,6 +15,13 @@ the paragraphs are an excerpt of the ad, handed over as `teaser`. A crawled ad
|
||||
("robot job") has no logo, so its company is what follows the title's last
|
||||
comma, or '' when there is none. An answer with no `<channel>` is not a search
|
||||
result and raises, since an empty list would read as a search that ran out.
|
||||
|
||||
The `<link>` opens Jobindex's `vis-job` page, and `posting_link` reads the
|
||||
"Se jobbet" button on it: the employer's site, The Hub or Jobindex's own
|
||||
`jobannonce`, checked on 2026-09-30. It is not part of the search: fetching
|
||||
that page counts as a click in Jobindex's statistics, so `app.jobs.posting`
|
||||
does it only for a role somebody opened. The page's "other ads" cards carry
|
||||
the same words on `/c?t=` redirects, which are not this ad and are skipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,7 +31,7 @@ import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
from urllib.parse import urlencode, urljoin, urlsplit
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
@@ -62,6 +69,8 @@ _AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | r
|
||||
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
|
||||
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
|
||||
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
||||
_SE_JOBBET = re.compile(r"<a\b([^>]*)>\s*Se jobbet\s*</a>", re.IGNORECASE)
|
||||
_HREF = re.compile(r'\bhref\s*=\s*"([^"]*)"', re.IGNORECASE)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
@@ -77,6 +86,20 @@ def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
||||
return out
|
||||
|
||||
|
||||
def posting_link(page_html: str) -> str:
|
||||
"""The "Se jobbet" link of a `vis-job` page: where the ad itself is, as an
|
||||
absolute URL, or '' when the page has none. A `/c?t=` link is another ad's
|
||||
redirect and is skipped."""
|
||||
for tag in _SE_JOBBET.finditer(page_html):
|
||||
found = _HREF.search(tag.group(1))
|
||||
if not found:
|
||||
continue
|
||||
url = urljoin("https://www.jobindex.dk/", html.unescape(found.group(1)).strip())
|
||||
if is_web_url(url) and urlsplit(url).path != "/c":
|
||||
return url
|
||||
return ""
|
||||
|
||||
|
||||
def _text(fragment: str) -> str:
|
||||
"""HTML to plain text. Jobindex breaks titles after a slash with a
|
||||
zero-width space, which would otherwise end up in `title_key`."""
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="da-DK">
|
||||
<head>
|
||||
<title>Løsningsarkitekt til Købstædernes Forsikring - 1702752 | Jobindex</title>
|
||||
</head>
|
||||
<body>
|
||||
<!-- Saved from a vis-job page on 2026-09-30, cut to the "Se jobbet" buttons and what follows them. -->
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-4">
|
||||
<div class="jix-toolbar__buttons">
|
||||
<span class="btn-list__element">
|
||||
|
||||
<span class="jobad-element-myjob" data-state="unsaved" data-tid="h1702752">
|
||||
<a class="btn btn-secondary btn-sm" href="/bruger/dine-job/h1702752/gem">
|
||||
<svg class="svg-icon svg-icon--mini m-0" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-button-myjob-unsaved"></use></svg>
|
||||
<span class="ml-1">Gem job</span>
|
||||
</a></span>
|
||||
</span>
|
||||
|
||||
<span class="btn-list__element d-none d-md-inline">
|
||||
<a class="btn btn-sm btn-primary seejobdesktop" data-click="/c?t=h1702752&ctx=w" href="https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring?promotion=2220670-jobindex" rel="noopener" target="_blank">Se jobbet</a>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
<a class="btn btn-sm btn-block btn-primary d-md-none mt-2 seejobmobil" data-click="/c?t=h1702752&ctx=w" href="https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring?promotion=2220670-jobindex" rel="noopener" target="_blank">Se jobbet</a>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="da-DK">
|
||||
<head>
|
||||
<title>Lead Software Engineer - Research Tech Delivery, AI & Digital Innovation - 14009457 | Jobindex</title>
|
||||
</head>
|
||||
<body>
|
||||
<!-- Saved from a vis-job page on 2026-09-30, cut to the "Se jobbet" buttons and what follows them. -->
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-4">
|
||||
<div class="jix-toolbar__buttons">
|
||||
<span class="btn-list__element">
|
||||
|
||||
<span class="jobad-element-myjob" data-state="unsaved" data-tid="r14009457">
|
||||
<a class="btn btn-secondary btn-sm" href="/bruger/dine-job/r14009457/gem">
|
||||
<svg class="svg-icon svg-icon--mini m-0" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-button-myjob-unsaved"></use></svg>
|
||||
<span class="ml-1">Gem job</span>
|
||||
</a></span>
|
||||
</span>
|
||||
|
||||
<span class="btn-list__element d-none d-md-inline">
|
||||
<a class="btn btn-sm btn-primary seejobdesktop" data-click="/c?t=r14009457&ctx=w" href="https://www.jobindex.dk/jobannonce/r14009457/lead-software-engineer-research-tech-delivery-ai-digital-innovation" rel="noopener" target="_blank">Se jobbet</a>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
<a class="btn btn-sm btn-block btn-primary d-md-none mt-2 seejobmobil" data-click="/c?t=r14009457&ctx=w" href="https://www.jobindex.dk/jobannonce/r14009457/lead-software-engineer-research-tech-delivery-ai-digital-innovation" rel="noopener" target="_blank">Se jobbet</a>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<aside class="offset-lg-1 col-md-3">
|
||||
|
||||
<div class="vp-card mb-4">
|
||||
<div class="vp-card__follow">
|
||||
<span class="vp-card__follow-count">22.134 følgere</span>
|
||||
<a href="/bruger/dine-virksomheder/258" id="widgets_follow_company_38e66c4c-e28a-46b6-825c-91496291af00" class="jix-follow-company" role="button" data-jix-login="user" data-jix-login-redirect="/bruger/dine-virksomheder/258"><span class="jix-follow-text-active">Følger</span><span class="jix-follow-text-inactive">Følg</span></a>
|
||||
</div>
|
||||
<a class="vp-card__name" href="/virksomhed/258/novo#om-virksomhed">Novo</a>
|
||||
<div class="vp-card__data">
|
||||
|
||||
<a class="vp-card__ratings" href="/virksomhed/258/novo#evalueringer">
|
||||
<div class="vp-card__rating-stars" title="5 ud af 5 stjerner"><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg><svg class="svg-icon svg-icon--sm vp-card__rating-icon vp-card__rating-icon--selected" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-rating-selected"></use></svg></div>
|
||||
<div class="vp-card__ratings-count">2671 evalueringer</div>
|
||||
</a> <div class="jix-link-block"><a href="/jobsoegning/virksomhed/258/novo"><span>67 jobannoncer</span><span aria-hidden="true"><svg class="svg-icon svg-icon--sm"><use xlink:href="#svg-linkarrow-right" xmlns:xlink="http://www.w3.org/1999/xlink"></use></svg></span></a></div>
|
||||
</div>
|
||||
|
||||
<a class="btn btn-secondary btn-sm btn-block" href="/virksomhed/258/novo#om-virksomhed">Om virksomheden</a>
|
||||
</div>
|
||||
|
||||
</aside>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
<section class="bg-surface-1 mt-4">
|
||||
<div class="recommended-jobs-container" id="recommended-jobs">
|
||||
<h2 class="mb-4">Anbefalede job</h2>
|
||||
<div class="job-card-container job-card-container--stacked">
|
||||
|
||||
|
||||
<article class="job-card" data-tid="h1700212"
|
||||
data-recommender-source="showjob"
|
||||
data-recommender-origin-tid="r14009457"
|
||||
>
|
||||
<div class="job-card__pubdate">
|
||||
Indrykket:
|
||||
<time datetime="2026-09-21">21. september 2026</time>
|
||||
</div>
|
||||
<a class="job-card__headline" href="/c?t=h1700212&ctx=w&ctt=r&u=22713329">
|
||||
Senior Embedded Engineer / Software Developer
|
||||
</a> <p class="job-card__company">Welltec A/S</p>
|
||||
|
||||
<div class="job-card__footer">
|
||||
<div class="job-card__area">
|
||||
<svg class="svg-icon svg-icon--sm"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-geolocation"></use></svg>
|
||||
<span class="job-card__area-text">Allerød</span>
|
||||
</div>
|
||||
<a class="btn btn-secondary btn-sm btn-block" href="/c?t=h1700212&ctx=w&ctt=r&u=22713329">Se jobbet</a>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
|
||||
|
||||
<article class="job-card" data-tid="h1695228"
|
||||
data-recommender-source="showjob"
|
||||
data-recommender-origin-tid="r14009457"
|
||||
>
|
||||
<div class="job-card__pubdate">
|
||||
Indrykket:
|
||||
<time datetime="2026-09-03">3. september 2026</time>
|
||||
</div>
|
||||
<a class="job-card__headline" href="/c?t=h1695228&ctx=w&ctt=r&u=22687519">
|
||||
Bliv vores nye Lead Software Engineer og vær med til at udvikle Nykredit-koncernens realkreditløsninger
|
||||
</a> <p class="job-card__company">Nykredit</p>
|
||||
|
||||
<div class="job-card__footer">
|
||||
<div class="job-card__area">
|
||||
<svg class="svg-icon svg-icon--sm"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-geolocation"></use></svg>
|
||||
<span class="job-card__area-text">Nordhavn</span>
|
||||
</div>
|
||||
<a class="btn btn-secondary btn-sm btn-block" href="/c?t=h1695228&ctx=w&ctt=r&u=22687519">Se jobbet</a>
|
||||
</div>
|
||||
</article>
|
||||
</body>
|
||||
</html>
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="da-DK">
|
||||
<head>
|
||||
<title>Senior Backend Developer - 14013846 | Jobindex</title>
|
||||
</head>
|
||||
<body>
|
||||
<!-- Saved from a vis-job page on 2026-09-30, cut to the "Se jobbet" buttons and what follows them. -->
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-4">
|
||||
<div class="jix-toolbar__buttons">
|
||||
<span class="btn-list__element">
|
||||
|
||||
<span class="jobad-element-myjob" data-state="unsaved" data-tid="r14013846">
|
||||
<a class="btn btn-secondary btn-sm" href="/bruger/dine-job/r14013846/gem">
|
||||
<svg class="svg-icon svg-icon--mini m-0" role="presentation"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-button-myjob-unsaved"></use></svg>
|
||||
<span class="ml-1">Gem job</span>
|
||||
</a></span>
|
||||
</span>
|
||||
|
||||
<span class="btn-list__element d-none d-md-inline">
|
||||
<a class="btn btn-sm btn-primary seejobdesktop" data-click="/c?t=r14013846&ctx=w" href="https://thehub.io/jobs/6ab5030dfd5518f38b6960b2?utm_source=jobindex&utm_medium=jobboard&utm_content=dk" rel="noopener" target="_blank">Se jobbet</a>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
<a class="btn btn-sm btn-block btn-primary d-md-none mt-2 seejobmobil" data-click="/c?t=r14013846&ctx=w" href="https://thehub.io/jobs/6ab5030dfd5518f38b6960b2?utm_source=jobindex&utm_medium=jobboard&utm_content=dk" rel="noopener" target="_blank">Se jobbet</a>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
<section class="bg-surface-1 mt-4">
|
||||
<div class="recommended-jobs-container" id="recommended-jobs">
|
||||
<h2 class="mb-4">Anbefalede job</h2>
|
||||
<div class="job-card-container job-card-container--stacked">
|
||||
|
||||
|
||||
<article class="job-card" data-tid="h1700172"
|
||||
data-recommender-source="showjob"
|
||||
data-recommender-origin-tid="r14013846"
|
||||
>
|
||||
<div class="job-card__pubdate">
|
||||
Indrykket:
|
||||
<time datetime="2026-09-21">21. september 2026</time>
|
||||
</div>
|
||||
<a class="job-card__headline" href="/c?t=h1700172&ctx=w&ctt=r&u=22713244">
|
||||
Senior Developer
|
||||
</a> <p class="job-card__company">Gubra A/S</p>
|
||||
|
||||
<div class="job-card__footer">
|
||||
<div class="job-card__area">
|
||||
<svg class="svg-icon svg-icon--sm"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-geolocation"></use></svg>
|
||||
<span class="job-card__area-text">Hørsholm</span>
|
||||
</div>
|
||||
<a class="btn btn-secondary btn-sm btn-block" href="/c?t=h1700172&ctx=w&ctt=r&u=22713244">Se jobbet</a>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
|
||||
|
||||
<article class="job-card" data-tid="h1696175"
|
||||
data-recommender-source="showjob"
|
||||
data-recommender-origin-tid="r14013846"
|
||||
>
|
||||
<div class="job-card__pubdate">
|
||||
Indrykket:
|
||||
<time datetime="2026-09-07">7. september 2026</time>
|
||||
</div>
|
||||
<a class="job-card__headline" href="/c?t=h1696175&ctx=w&ctt=r&u=22692389">
|
||||
Senior .NET Developer - Microservices & Integrations
|
||||
</a> <p class="job-card__company">twoday A/S</p>
|
||||
|
||||
<div class="job-card__footer">
|
||||
<div class="job-card__area">
|
||||
<svg class="svg-icon svg-icon--sm"><use xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#svg-geolocation"></use></svg>
|
||||
<span class="job-card__area-text">Nordhavn</span>
|
||||
</div>
|
||||
<a class="btn btn-secondary btn-sm btn-block" href="/c?t=h1696175&ctx=w&ctt=r&u=22692389">Se jobbet</a>
|
||||
</div>
|
||||
</article>
|
||||
</body>
|
||||
</html>
|
||||
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
||||
"VALUES (?, ?, '', '')", (key, name))
|
||||
conn.execute("PRAGMA user_version = 19")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25]
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26]
|
||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(reopened.companies()) == 3
|
||||
|
||||
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
||||
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
|
||||
conn.execute("PRAGMA user_version = 20")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25]
|
||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26]
|
||||
# The tag of 21 is All countries and All roles since 24.
|
||||
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
|
||||
|
||||
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25]
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||
|
||||
@@ -437,7 +437,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
|
||||
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
|
||||
conn.execute("PRAGMA user_version = 22")
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [23, 24, 25]
|
||||
assert migrated.migrations_applied == [23, 24, 25, 26]
|
||||
assert migrated.agents()[0].fetch_terms == []
|
||||
|
||||
|
||||
|
||||
@@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
|
||||
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
|
||||
conn.execute("PRAGMA user_version = 23")
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [24, 25]
|
||||
assert migrated.migrations_applied == [24, 25, 26]
|
||||
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
|
||||
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
|
||||
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""`fetch_posting`: a role's posting text on demand, and for a Jobindex role
|
||||
the page its "Se jobbet" button leads to. No network: `scrape` is stubbed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.jobs import posting
|
||||
from app.storage import Store
|
||||
from desk.models import Item
|
||||
from desk.tools import scrape
|
||||
from desk.tools.article import ArticleContent
|
||||
from desk.tools.checks import Page
|
||||
from prepped import upsert
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
VIS_JOB = "https://www.jobindex.dk/vis-job/h1702752"
|
||||
LINK = ("https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring"
|
||||
"?promotion=2220670-jobindex")
|
||||
PAGE = (ROOT / "tests" / "fixtures" / "jobindex" / "vis-job-employer.html").read_text("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "roledesk.db")
|
||||
|
||||
|
||||
def role(store: Store, url: str = "https://boards.example/acme/1", source_id: str = "1",
|
||||
**raw: object) -> str:
|
||||
item = Item(source_id=source_id, location=url, title="Engineer", body="A teaser.",
|
||||
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
|
||||
item.source = "ats"
|
||||
upsert(store, item)
|
||||
return item.uid
|
||||
|
||||
|
||||
def jobindex_role(store: Store) -> str:
|
||||
return role(store, VIS_JOB, provider="jobindex")
|
||||
|
||||
|
||||
def content(text: str, url: str = "") -> scrape.Scraped:
|
||||
return scrape.Scraped(ArticleContent(format="plain", content=text, source_url=url,
|
||||
method="test"), "", "http", ())
|
||||
|
||||
|
||||
def failed(category: str) -> scrape.Scraped:
|
||||
return scrape.Scraped(None, category, "http", ())
|
||||
|
||||
|
||||
class Web:
|
||||
"""Stands in for the network: `pages` maps a URL to what `scrape.HTTP`
|
||||
answers with, `texts` a URL to what the article of that page reads as."""
|
||||
|
||||
def __init__(self, monkeypatch: pytest.MonkeyPatch, pages: dict[str, str],
|
||||
texts: dict[str, str]) -> None:
|
||||
self.asked: list[str] = []
|
||||
self.pages, self.texts = pages, texts
|
||||
monkeypatch.setattr(scrape, "HTTP", scrape.Getter("http", self.fetch))
|
||||
monkeypatch.setattr(scrape, "STANDARD", {"http": scrape.HTTP})
|
||||
monkeypatch.setattr(scrape, "scrape", self.scrape)
|
||||
|
||||
def fetch(self, url: str) -> Page | None:
|
||||
self.asked.append(url)
|
||||
html = self.pages.get(url)
|
||||
return None if html is None else Page(status=200, html=html, url=url, headers={})
|
||||
|
||||
def scrape(self, url: str, getters: object = None, first: str = "http") -> scrape.Scraped:
|
||||
if getters is not None: # the page is fetched by the getter it was handed
|
||||
page = getters["http"].fetch(url) # type: ignore[index]
|
||||
return content(self.texts[url], url) if page and url in self.texts else failed("empty")
|
||||
self.asked.append(url)
|
||||
return content(self.texts[url], url) if url in self.texts else failed("not_found")
|
||||
|
||||
|
||||
def test_a_role_that_cannot_be_fetched_says_why(store: Store,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
Web(monkeypatch, {}, {})
|
||||
with pytest.raises(posting.UnknownRole) as unknown:
|
||||
posting.fetch_posting(store, "nope")
|
||||
assert (unknown.value.status, str(unknown.value)) == (404, "no such role: nope")
|
||||
|
||||
bare = role(store, url="")
|
||||
with pytest.raises(posting.NoLink) as nolink:
|
||||
posting.fetch_posting(store, bare)
|
||||
assert nolink.value.status == 422
|
||||
|
||||
|
||||
def test_an_ordinary_role_is_scraped_from_its_link_once(store: Store,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = role(store)
|
||||
web = Web(monkeypatch, {}, {})
|
||||
with pytest.raises(posting.Unreadable) as gone:
|
||||
posting.fetch_posting(store, uid)
|
||||
assert gone.value.status == 422 and "gone" in str(gone.value)
|
||||
assert web.asked == ["https://boards.example/acme/1"]
|
||||
|
||||
web.texts["https://boards.example/acme/1"] = " The whole posting.\n"
|
||||
got = posting.fetch_posting(store, uid)
|
||||
assert got.article_text == "The whole posting." and got.article_at
|
||||
assert got.posting_url == ""
|
||||
with pytest.raises(posting.AlreadyFetched) as again:
|
||||
posting.fetch_posting(store, uid)
|
||||
assert again.value.status == 409
|
||||
|
||||
|
||||
def test_an_unknown_failure_reads_as_nothing_readable(store: Store,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = role(store)
|
||||
web = Web(monkeypatch, {}, {})
|
||||
monkeypatch.setattr(web, "scrape", lambda url, *a, **k: failed("weird"))
|
||||
monkeypatch.setattr(scrape, "scrape", web.scrape)
|
||||
with pytest.raises(posting.Unreadable, match="Nothing readable"):
|
||||
posting.fetch_posting(store, uid)
|
||||
|
||||
|
||||
def test_a_jobindex_role_follows_its_se_jobbet_link_and_keeps_it(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = jobindex_role(store)
|
||||
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad.", LINK: "The real posting."})
|
||||
got = posting.fetch_posting(store, uid)
|
||||
assert got.article_text == "The real posting."
|
||||
assert got.posting_url == LINK and got.original_url == VIS_JOB
|
||||
assert web.asked == [VIS_JOB, LINK]
|
||||
stored = store.role(uid)
|
||||
assert stored is not None and stored.posting_url == LINK
|
||||
|
||||
|
||||
def test_a_jobindex_role_falls_back_to_the_ads_own_text(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# The link gives nothing readable: the link is kept, the vis-job text stored.
|
||||
uid = jobindex_role(store)
|
||||
web = Web(monkeypatch, {VIS_JOB: PAGE}, {VIS_JOB: "The Jobindex ad."})
|
||||
got = posting.fetch_posting(store, uid)
|
||||
assert got.article_text == "The Jobindex ad." and got.posting_url == LINK
|
||||
assert web.asked == [VIS_JOB, LINK]
|
||||
|
||||
# No button on the page: the ad's own text, no link, one request.
|
||||
other = role(store, "https://www.jobindex.dk/vis-job/r2", "2", provider="jobindex")
|
||||
web = Web(monkeypatch, {"https://www.jobindex.dk/vis-job/r2": "<p>Ingen knap</p>"},
|
||||
{"https://www.jobindex.dk/vis-job/r2": "Ad two."})
|
||||
got = posting.fetch_posting(store, other)
|
||||
assert got.article_text == "Ad two." and got.posting_url == ""
|
||||
assert web.asked == ["https://www.jobindex.dk/vis-job/r2"]
|
||||
|
||||
|
||||
def test_a_jobindex_role_whose_pages_read_as_nothing_is_unreadable_but_keeps_the_link(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = jobindex_role(store)
|
||||
Web(monkeypatch, {VIS_JOB: PAGE}, {})
|
||||
with pytest.raises(posting.Unreadable) as error:
|
||||
posting.fetch_posting(store, uid)
|
||||
assert error.value.failure == "not_found"
|
||||
stored = store.role(uid)
|
||||
assert stored is not None and stored.posting_url == LINK and stored.article_text == ""
|
||||
|
||||
|
||||
def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = jobindex_role(store)
|
||||
store.set_posting_url(uid, LINK)
|
||||
web = Web(monkeypatch, {}, {LINK: "The real posting."})
|
||||
assert posting.fetch_posting(store, uid).article_text == "The real posting."
|
||||
assert web.asked == [LINK]
|
||||
+1
-1
@@ -314,7 +314,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [22, 23, 24, 25]
|
||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26]
|
||||
one = row(migrated, "1")
|
||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||
|
||||
@@ -100,3 +100,39 @@ def test_an_answer_without_a_channel_raises() -> None:
|
||||
def test_saved_params_keep_the_query_and_a_real_area() -> None:
|
||||
assert jobindex.clean_params({"q": "x", "area": "3"}) == {"q": "x", "area": 3}
|
||||
assert jobindex.clean_params({"q": "x", "area": 0}) == {"q": "x"}
|
||||
|
||||
|
||||
# ------------------------------------------------- the link out of a vis-job page
|
||||
|
||||
PAGES = ROOT / "tests" / "fixtures" / "jobindex"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name, link", [
|
||||
("vis-job-employer.html",
|
||||
"https://careers.kfforsikring.dk/jobs/8456088-loesningsarkitekt-til-koebstaedernes-forsikring"
|
||||
"?promotion=2220670-jobindex"),
|
||||
("vis-job-thehub.html",
|
||||
"https://thehub.io/jobs/6ab5030dfd5518f38b6960b2"
|
||||
"?utm_source=jobindex&utm_medium=jobboard&utm_content=dk"),
|
||||
("vis-job-jobannonce.html",
|
||||
"https://www.jobindex.dk/jobannonce/r14009457/"
|
||||
"lead-software-engineer-research-tech-delivery-ai-digital-innovation"),
|
||||
])
|
||||
def test_posting_link_is_the_se_jobbet_button_of_a_saved_page(name: str, link: str) -> None:
|
||||
assert jobindex.posting_link((PAGES / name).read_text(encoding="utf-8")) == link
|
||||
|
||||
|
||||
def test_posting_link_skips_other_ads_and_is_empty_without_a_button() -> None:
|
||||
other = '<a class="btn" href="/c?t=h1&ctx=w&ctt=r&u=2">Se jobbet</a>'
|
||||
assert jobindex.posting_link(other) == ""
|
||||
assert jobindex.posting_link("<p>Ingen knap</p>") == ""
|
||||
assert jobindex.posting_link("") == ""
|
||||
mine = '<A HREF="https://a.example/x?a=1&b=2" rel="noopener">\n Se jobbet </A>'
|
||||
assert jobindex.posting_link(other + mine) == "https://a.example/x?a=1&b=2"
|
||||
|
||||
|
||||
def test_posting_link_is_absolute_and_never_a_script() -> None:
|
||||
assert jobindex.posting_link('<a href="/jobannonce/r1/x">Se jobbet</a>') \
|
||||
== "https://www.jobindex.dk/jobannonce/r1/x"
|
||||
assert jobindex.posting_link('<a href="javascript:alert(1)">Se jobbet</a>') == ""
|
||||
assert jobindex.posting_link('<a class="x">Se jobbet</a>') == ""
|
||||
+37
-7
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 25
|
||||
assert LATEST == 26
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||
@@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||
Prompt(name="Backend", text="python backend")]
|
||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||
@@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||
@@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25]
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||
"SELECT uid, place_fold, seniority FROM roles")}
|
||||
@@ -688,7 +688,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25]
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||
assert migrated.role_places(stated.uid) == []
|
||||
@@ -730,7 +730,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [25]
|
||||
assert migrated.migrations_applied == [25, 26]
|
||||
with sqlite3.connect(path) as conn:
|
||||
found = {sid: rest for sid, *rest in conn.execute(
|
||||
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
|
||||
@@ -738,6 +738,36 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
|
||||
assert found == {sid: list(after) for sid, (_, after) in cases.items()}
|
||||
|
||||
|
||||
def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path) -> None:
|
||||
path = tmp_path / "v25.db"
|
||||
store = Store(path)
|
||||
upsert(store, posting("a"))
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN posting_url")
|
||||
conn.execute("PRAGMA user_version = 25")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [26]
|
||||
role = migrated.role(posting("a").uid)
|
||||
assert role is not None and role.posting_url == ""
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 25") # the column is there already
|
||||
assert Store(path).migrations_applied == [26]
|
||||
|
||||
|
||||
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
|
||||
upsert(store, posting("a"))
|
||||
uid = posting("a").uid
|
||||
assert store.set_posting_url(uid, "https://careers.example/jobs/1")
|
||||
assert not store.set_posting_url("nope", "https://careers.example/jobs/1")
|
||||
upsert(store, posting("a")) # the next pull reads the same role again
|
||||
role = store.role(uid)
|
||||
assert role is not None and role.posting_url == "https://careers.example/jobs/1"
|
||||
row = next(r for r in store.roles_page() if r.uid == uid)
|
||||
assert row.posting_url == "https://careers.example/jobs/1"
|
||||
|
||||
|
||||
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
||||
upsert(store, posting(place="Berlin", remote=True, salary_min="80000",
|
||||
salary_max=95000, currency="EUR", department="Eng"))
|
||||
|
||||
+3
-2
@@ -1224,7 +1224,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
|
||||
assert set(row) == {
|
||||
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
|
||||
"seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state",
|
||||
"original_url", "posted_at", "score", "gated_reason", "prompt", "vote",
|
||||
"original_url", "posting_url", "posted_at", "score", "gated_reason", "prompt", "vote",
|
||||
"voted_at", "read_at", "tags", "excerpt", "copies",
|
||||
}
|
||||
assert row["uid"] == uid
|
||||
@@ -1734,7 +1734,8 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
|
||||
assert kept["dup_of"] is None
|
||||
assert kept["copies"] == [{
|
||||
"uid": search.uid, "host": "jobindex", "place": "Ballerup, Denmark",
|
||||
"original_url": search.location, "posted_at": kept["copies"][0]["posted_at"]}]
|
||||
"original_url": search.location, "posting_url": "",
|
||||
"posted_at": kept["copies"][0]["posted_at"]}]
|
||||
copy = client.get(f"/api/roles/{search.uid}").json()
|
||||
assert copy["dup_of"] == board.uid and copy["copies"] == []
|
||||
|
||||
|
||||
Reference in new issue
Block a user