From 43c0bc7523c65ec254bf72524e4b7bf13c20792b Mon Sep 17 00:00:00 2001 From: Wolf Date: Sat, 3 Oct 2026 12:18:56 +0200 Subject: [PATCH] Read Akademikernes Jobbank's search feed, with each ad's text from its own page Co-Authored-By: Claude Sonnet 5.5 --- plugins/readers/jobbank.py | 253 +++++++++++++++++++++ tests/fixtures/search/jobbank-detail.html | 42 ++++ tests/fixtures/search/jobbank-empty.xml | 10 + tests/fixtures/search/jobbank-places.xml | 65 ++++++ tests/fixtures/search/jobbank.xml | 44 ++++ tests/test_readers_jobbank.py | 256 ++++++++++++++++++++++ 6 files changed, 670 insertions(+) create mode 100644 plugins/readers/jobbank.py create mode 100644 tests/fixtures/search/jobbank-detail.html create mode 100644 tests/fixtures/search/jobbank-empty.xml create mode 100644 tests/fixtures/search/jobbank-places.xml create mode 100644 tests/fixtures/search/jobbank.xml create mode 100644 tests/test_readers_jobbank.py diff --git a/plugins/readers/jobbank.py b/plugins/readers/jobbank.py new file mode 100644 index 0000000..b889b2d --- /dev/null +++ b/plugins/readers/jobbank.py @@ -0,0 +1,253 @@ +"""Akademikernes Jobbank (jobbank.dk): Denmark's portal for academics and +researchers, with corporate ads (LEO Pharma, Ørsted) beside the public sector, +searched through the RSS feed of its search, `/job/rss/`; no key. + +Checked live on 2026-10-03. The trailing slash matters, `/job/rss` answers 301. +`key` is the query, words are OR. `amt` is the region of the site's own search +form (2 is Storkøbenhavn), `fjernarbejde` is `helt` or `delvist`. 100 items a +page and an empty channel ends the list. The feed states no total and is not +sorted by date, so a search is never complete and the newest ads may be on any +page. + +An item is `` the role, a `<link>` to the ad's page on jobbank.dk and a +`<description>` of one line: "Fuldtidsjob hos LEO PHARMA A/S, Storkøbenhavn, +Fyn (Ansøgningsfrist: 18.10.2026)", the kind of job, the employer, up to three +of the regions the ad is for (the rest as "+12 andre", which names none) and +the deadline or "løbende". The employer may hold a comma, so the regions are +taken from the end, as far as they are regions of the form. The feed has no +text: `detail` reads the ad's own page, whose JSON-LD `JobPosting` has the +description, the town and the dates. The page's "Ansøg" button goes through +`/fwd/webjob/`, which robots.txt disallows and which is never requested; a +Cloudflare challenge in place of the page is an error for that role, not an +empty text. + +`fjernarbejde` is a word, not a flag, so the saved param stays in `Ask.params` +(the runner takes a bool only for the `remote` dimension) and `page` reads it +there. A search asked for `helt` lists remote roles only, and its places say so. +""" + +from __future__ import annotations + +import json +import re +import xml.etree.ElementTree as ET +from collections.abc import Callable +from typing import Any +from urllib.parse import urlencode, urlsplit + +from ._contract import Ask, Fields, Filters, Place, Reads +from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt, place + +NAME = "jobbank" +LABEL = "Akademikernes Jobbank (Denmark)" + +READS = Reads( + scope="search", listing="paged", page_size=100, ceiling=2000, + text="detail", complete="never", + filters=Filters(query="one", place="one", remote="one"), + places="many", + fields=Fields(posted="trusted", country="stated"), + robots="read: robots.txt allows /job/ and disallows /fwd/webjob/, /link/, /redirect/ " + "and /ajax/ with Host-load 2; the reader requests /job/rss/ and /job/<id>/ only " + "and never follows /fwd/webjob/ (checked 2026-10-03)") + +#: Saved param -> the filter dimension it fills. +PARAMS = {"q": "query", "region": "place", "remote": "remote"} + +RSS = "https://jobbank.dk/job/rss/" + +#: The `amt` ids of the site's own search form, with its labels. Empty is all of +#: Denmark and not a key here. "Øresundsregionen" and the "Udlandet" ids are +#: places beyond Denmark's own regions. +REGIONS: dict[str, str] = { + "2": "Storkøbenhavn", "3": "Nordsjælland", "14": "Østsjælland", "4": "Vestsjælland", + "5": "Sydsjælland & Øer", "13": "Fyn", "12": "Sønderjylland", + "11": "Sydvestjylland (Esbjerg)", "9": "Vestjylland", "10": "Sydøstjylland", + "7": "Midtjylland", "8": "Østjylland (Aarhus)", "6": "Nordjylland", "20": "Bornholm", + "21": "Øresundsregionen", "22": "Grønland & Færøerne", "23": "Udlandet (Sverige)", + "24": "Udlandet (Norge)", "19": "Udlandet (øvrige)", +} + +#: The `fjernarbejde` values of the form: fully and partly remote. +REMOTE: dict[str, str] = {"helt": "Fully remote", "delvist": "Partly remote"} + +_LABELS = frozenset(REGIONS.values()) + +#: The country a region lies in, where it lies in one. A role with every listed +#: region in one country has that country; "Øresundsregionen", "Grønland & +#: Færøerne" and "Udlandet (øvrige)" name none. +_COUNTRY: dict[str, str] = { + label: "DK" for label in REGIONS.values() + if label not in ("Øresundsregionen", "Grønland & Færøerne") and not label.startswith("Udlandet") +} | {"Udlandet (Sverige)": "SE", "Udlandet (Norge)": "NO"} + +_ID = re.compile(r"/job/(\d+)(?:/|$)") +#: "Fuldtidsjob, Graduate/trainee hos LEO PHARMA A/S, Storkøbenhavn +2 andre +#: (Ansøgningsfrist: 18.10.2026)": the kind, the employer and regions, the deadline. +_LINE = re.compile(r"(?P<kind>.*?) hos (?P<rest>.*?)\s*" + r"(?:\(Ansøgningsfrist:\s*(?P<deadline>[^)]*)\))?\s*", re.DOTALL) +_MORE = re.compile(r"\s*\+\d+ andre$") +_DEADLINE = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})") +_JSON_LD = re.compile(r"<script[^>]*application/ld\+json[^>]*>(.*?)</script>", + re.IGNORECASE | re.DOTALL) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only("jobbank.dk") + + +def clean_params(params: dict[str, Any]) -> dict[str, Any]: + """The saved params of a search: `q`, `region` when it is the id of a + region of the form and `remote` when it is `helt` or `delvist`.""" + out: dict[str, Any] = {"q": params["q"]} + region = str(params.get("region") or "").strip() + if region: + if region not in REGIONS: + raise ValueError("region is empty or the id of a region of jobbank.dk, " + "2 is Storkøbenhavn") + out["region"] = region + remote = str(params.get("remote") or "").strip() + if remote: + if remote not in REMOTE: + raise ValueError("remote is empty, helt or delvist") + out["remote"] = remote + return out + + +def _query(ask: Ask, number: int) -> tuple[str, str]: + """The feed's URL for page `number`, and the `fjernarbejde` it asks for.""" + region = str(ask.places[0] if ask.places else ask.params.get("region") or "") + remote = str(ask.params.get("remote") or "") + if region and region not in REGIONS: + raise ValueError(f"jobbank region {region!r} is not one of the ids in REGIONS") + if remote and remote not in REMOTE: + raise ValueError(f"jobbank remote {remote!r} is not one of the words in REMOTE") + query: dict[str, Any] = {"key": ask.query or str(ask.params.get("q") or "")} + if region: + query["amt"] = region + if remote: + query["fjernarbejde"] = remote + return RSS + "?" + urlencode(query | {"page": number}), remote + + +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: + """One page of a search; the cursor is the page number. The next page + follows a full page (counted before any item is dropped, so a dropped item + does not pass for the end) while the ceiling is not reached.""" + number = int(cursor or 1) + url, remote = _query(ask, number) + try: + root = ET.fromstring(get(url)) + except ET.ParseError as exc: + raise ValueError(f"jobbank answered something that is not RSS ({exc}), " + "a challenge in place of the feed?") from exc + channel = root.find("channel") + if channel is None: + raise ValueError("jobbank answered without an RSS channel") + items = [] + for item in channel.findall("item"): + link = (item.findtext("link") or "").strip() + found = _ID.search(link) + items.append({ + "id": found.group(1) if found else "", + "link": link, + "title": (item.findtext("title") or "").strip(), + "description": (item.findtext("description") or "").strip(), + "published": item.findtext("pubDate"), + }) + decoded = {"remote": remote, "items": items} + size, ceiling = READS.page_size or 100, READS.ceiling or 2000 + more = len(items) >= size and number * size < ceiling + return decoded, [{"id": item["id"]} for item in items], None, number + 1 if more else None + + +def _regions(rest: str) -> tuple[str, list[str], bool]: + """`(company, regions, cut)` of what follows "hos": the regions are the + known labels at the end, "+12 andre" (`cut`) the ones the feed leaves out. + Nothing but the employer is left when no label is at the end.""" + cut = bool(_MORE.search(rest)) + parts = _MORE.sub("", rest).split(", ") + regions: list[str] = [] + while len(parts) > 1 and parts[-1].strip() in _LABELS: + regions.insert(0, parts.pop().strip()) + return ", ".join(parts).strip(), regions, cut + + +def _country(regions: list[str], cut: bool) -> str | None: + """The one country all the listed regions lie in; None when they lie in + several or in none, or when "+N andre" left some out.""" + found = {_COUNTRY.get(region) for region in regions} + return found.pop() if len(found) == 1 and not cut and None not in found else None + + +def rows(decoded: Any, slug: str) -> list[Row]: + out: list[Row] = [] + for item in decoded["items"]: + link, title = item["link"], item["title"] + if not title or not item["id"] or not is_web_url(link): + continue + line = _LINE.fullmatch(item["description"]) + kind = line.group("kind").strip() if line else "" + company, regions, cut = _regions(line.group("rest")) if line else ("", [], False) + places: list[Place] = [place(region, True if decoded["remote"] == "helt" else None) + for region in regions] + row: Row = { + "source_id": item["id"], + "title": title, + "company": company, + "url": link, + "places": places, + "location": regions[0] if regions else "", + "posted": parse_dt(item["published"]), + } + if country := _country(regions, cut): + row["country"] = country + if kind: + row["jobType"] = kind + if line and (day := _DEADLINE.fullmatch((line.group("deadline") or "").strip())): + row["applicationDeadline"] = f"{day.group(3)}-{day.group(2)}-{day.group(1)}" + out.append(row) + return out + + +def _posting(body: bytes) -> dict[str, Any]: + """The JSON-LD `JobPosting` of an ad's page. Raises ValueError when the + page has none, as a Cloudflare challenge page has not.""" + for found in _JSON_LD.finditer(body.decode("utf-8", errors="replace")): + try: + data = json.loads(found.group(1), strict=False) + except ValueError: + continue + for node in data if isinstance(data, list) else [data]: + if isinstance(node, dict) and "JobPosting" in ( + node["@type"] if isinstance(node.get("@type"), list) else [node.get("@type")]): + return node + raise ValueError("jobbank answered a page without a JobPosting, a challenge in place of the ad?") + + +def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: + """The ad's own page: the description, the town added to the places the + feed named, and the date when the feed gave none. Only a `/job/` page of + jobbank.dk is requested, never `/fwd/webjob/`. `validThrough` is not + taken: the feed's deadline is already in `raw`, and `detail` may fill + `_runner.DETAIL_FIELDS` only.""" + url = row["url"] + parts = urlsplit(url) + if parts.hostname != "jobbank.dk" or not parts.path.startswith("/job/"): + raise ValueError(f"jobbank reads /job/ pages of jobbank.dk only, not {url}") + posting = _posting(get(url)) + out: dict[str, Any] = {} + if description := html_text(posting.get("description")): + out["description"] = description + location = posting.get("jobLocation") + if isinstance(location, list): + location = location[0] if location else None + address = location.get("address") if isinstance(location, dict) else None + town = str(address.get("addressLocality") or "").strip() if isinstance(address, dict) else "" + if town: + found: list[Place] = [dict(p) for p in row.get("places") or []] # type: ignore[misc] + add_place(found, place(town)) + out["places"] = found + if not row.get("posted") and (posted := parse_dt(posting.get("datePosted"))): + out["posted"] = posted + return out diff --git a/tests/fixtures/search/jobbank-detail.html b/tests/fixtures/search/jobbank-detail.html new file mode 100644 index 0000000..8041613 --- /dev/null +++ b/tests/fixtures/search/jobbank-detail.html @@ -0,0 +1,42 @@ +<!DOCTYPE HTML> +<html lang="da"> +<head> +<meta charset="utf-8"> +<title>Data scientister og historiske datamanagere til Rigsarkivet - Akademikernes Jobbank + + +

Data scientister og historiske datamanagere til Rigsarkivet

Ansøg + diff --git a/tests/fixtures/search/jobbank-empty.xml b/tests/fixtures/search/jobbank-empty.xml new file mode 100644 index 0000000..55b930d --- /dev/null +++ b/tests/fixtures/search/jobbank-empty.xml @@ -0,0 +1,10 @@ + + + Job fra Akademikernes Jobbank + https://jobbank.dk + 0 nyeste job og karrieremuligheder fra Akademikernes Jobbank. + DA + + + + diff --git a/tests/fixtures/search/jobbank-places.xml b/tests/fixtures/search/jobbank-places.xml new file mode 100644 index 0000000..80c4325 --- /dev/null +++ b/tests/fixtures/search/jobbank-places.xml @@ -0,0 +1,65 @@ + + + Job fra Akademikernes Jobbank + https://jobbank.dk + 100 nyeste job og karrieremuligheder fra Akademikernes Jobbank. + DA + + + <![CDATA[Data scientister og historiske datamanagere til Rigsarkivet]]> + + https://jobbank.dk/job/3111914/rigsarkivet-staten/data-scientister-og-historiske + https://jobbank.dk/job/3111914/rigsarkivet-staten/data-scientister-og-historiske + Fri, 04 Sep 2026 00:00:00 +0200 + + + <![CDATA[Global Program Director]]> + + https://jobbank.dk/job/3121042/leo-pharma-as/global-program-director + https://jobbank.dk/job/3121042/leo-pharma-as/global-program-director + Mon, 21 Sep 2026 00:00:00 +0200 + + + <![CDATA[Principal Scientist, Formulation Development]]> + + https://jobbank.dk/job/3124549/leo-pharma-as/principal-scientist-formulati + https://jobbank.dk/job/3124549/leo-pharma-as/principal-scientist-formulati + Wed, 23 Sep 2026 00:00:00 +0200 + + + <![CDATA[Teamlead Reliability Engineers]]> + + https://jobbank.dk/job/3122426/orsted/teamlead-reliability-engineers + https://jobbank.dk/job/3122426/orsted/teamlead-reliability-engineers + Fri, 02 Oct 2026 00:00:00 +0200 + + + <![CDATA[Head of Wholesale - Nordics & Baltics]]> + + https://jobbank.dk/job/3131081/samsoe-samsoe/head-of-wholesale-nordics + https://jobbank.dk/job/3131081/samsoe-samsoe/head-of-wholesale-nordics + Fri, 02 Oct 2026 00:00:00 +0200 + + + <![CDATA[Environmental Data & Carbon Footprint Specialist]]> + + https://jobbank.dk/job/3119220/elopak/environmental-data-carbon-fo + https://jobbank.dk/job/3119220/elopak/environmental-data-carbon-fo + Fri, 02 Oct 2026 00:00:00 +0200 + + + <![CDATA[Mechanical Lead - Data Centres]]> + + https://jobbank.dk/job/3035074/cowi-as/mechanical-lead-data-centres + https://jobbank.dk/job/3035074/cowi-as/mechanical-lead-data-centres + Mon, 21 Sep 2026 00:00:00 +0200 + + + <![CDATA[Advisory AI Data Specialist]]> + + https://jobbank.dk/job/3112442/dell-technologies/advisory-ai-data-specialist + https://jobbank.dk/job/3112442/dell-technologies/advisory-ai-data-specialist + Fri, 04 Sep 2026 00:00:00 +0200 + + + diff --git a/tests/fixtures/search/jobbank.xml b/tests/fixtures/search/jobbank.xml new file mode 100644 index 0000000..75d7adf --- /dev/null +++ b/tests/fixtures/search/jobbank.xml @@ -0,0 +1,44 @@ + + + Job fra Akademikernes Jobbank + https://jobbank.dk + 12 nyeste job og karrieremuligheder fra Akademikernes Jobbank. + DA + + + <![CDATA[Country Regulatory Specialist til Nordisk Alkali]]> + + https://jobbank.dk/job/3121464/jobadvice-management/country-regulatory-specialist + https://jobbank.dk/job/3121464/jobadvice-management/country-regulatory-specialist + Mon, 28 Sep 2026 00:00:00 +0200 + + + <![CDATA[Fagleder (el-installationer)]]> + + https://jobbank.dk/job/3068603/wsp/fagleder-el-installationer + https://jobbank.dk/job/3068603/wsp/fagleder-el-installationer + Fri, 02 Oct 2026 00:00:00 +0200 + + + <![CDATA[Certificeret statiker]]> + + https://jobbank.dk/job/3068604/wsp/certificeret-statiker + https://jobbank.dk/job/3068604/wsp/certificeret-statiker + Fri, 02 Oct 2026 00:00:00 +0200 + + + <![CDATA[Infrastruktur konsulent]]> + + https://jobbank.dk/job/3069149/itm8/infrastruktur-konsulent + https://jobbank.dk/job/3069149/itm8/infrastruktur-konsulent + Thu, 01 Oct 2026 00:00:00 +0200 + + + <![CDATA[Sales Specialist, Proteomics Sciences – Denmark]]> + + https://jobbank.dk/job/3122957/thermo-fisher-scient/sales-specialist-proteomics-s + https://jobbank.dk/job/3122957/thermo-fisher-scient/sales-specialist-proteomics-s + Thu, 01 Oct 2026 00:00:00 +0200 + + + diff --git a/tests/test_readers_jobbank.py b/tests/test_readers_jobbank.py new file mode 100644 index 0000000..bc7d041 --- /dev/null +++ b/tests/test_readers_jobbank.py @@ -0,0 +1,256 @@ +"""The Akademikernes Jobbank reader: the RSS feed of a search, 100 items a page, +and the JSON-LD of an ad's own page for its text. No network: `get` is a stub.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlsplit + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import readers # noqa: E402 +from plugins.readers import Ask, jobbank # noqa: E402 + +FIXTURES = ROOT / "tests" / "fixtures" / "search" +PLACES = (FIXTURES / "jobbank-places.xml").read_bytes() +REMOTE = (FIXTURES / "jobbank.xml").read_bytes() +EMPTY = (FIXTURES / "jobbank-empty.xml").read_bytes() +AD = (FIXTURES / "jobbank-detail.html").read_bytes() + + +def _rows(body: bytes = PLACES, ask: Ask | None = None) -> list[dict[str, Any]]: + decoded, _results, _total, _cursor = jobbank.page( + lambda url, **kw: body, "", None, ask or Ask(params={"q": "x"})) + return list(jobbank.rows(decoded, "")) + + +def _feed(items: int) -> bytes: + body = "".join(f"Role {n}Fuldtidsjob hos Acme A/S, " + f"Fyn (Ansøgningsfrist: løbende)" + f"https://jobbank.dk/job/{n + 1}/acme/role" + for n in range(items)) + return f"Jobbank{body}".encode() + + +def _item(description: str) -> bytes: + return (f"Role" + "https://jobbank.dk/job/7/acme/role" + "").encode() + + +def test_it_is_a_search_reader_with_the_robots_decision() -> None: + assert readers.PROVIDERS["jobbank"] is jobbank + reads = jobbank.READS + assert (reads.scope, reads.page_size, reads.ceiling, reads.text, reads.complete) == ( + "search", 100, 2000, "detail", "never") + assert "/job/" in reads.robots and "/fwd/webjob/" in reads.robots + assert jobbank.allowed("")("jobbank.dk") and not jobbank.allowed("")("www.jobbank.dk") + + +def test_the_regions_are_the_forms_and_the_remote_words_are_its_two() -> None: + assert jobbank.REGIONS["2"] == "Storkøbenhavn" and len(jobbank.REGIONS) == 19 + assert set(jobbank.REMOTE) == {"helt", "delvist"} + assert jobbank.PARAMS == {"q": "query", "region": "place", "remote": "remote"} + + +def test_it_reads_title_company_regions_date_and_deadline() -> None: + first, leo, phd, orsted, *_ = _rows() + assert first["source_id"] == "3111914" + assert first["url"] == ("https://jobbank.dk/job/3111914/rigsarkivet-staten/" + "data-scientister-og-historiske") + assert first["title"].startswith("Data scientister og historiske") + assert first["company"] == "Rigsarkivet - Statens Arkiver" + assert [p["text"] for p in first["places"]] == ["Storkøbenhavn", "Fyn"] + assert first["location"] == "Storkøbenhavn" and first["country"] == "DK" + assert first["posted"].date().isoformat() == "2026-09-04" + assert (first["jobType"], first["applicationDeadline"]) == ("Fuldtidsjob", "2026-10-07") + assert "applicationDeadline" not in leo and leo["company"] == "LEO PHARMA A/S" + assert phd["jobType"] == "Ph.d. & Postdoc" + assert [p["text"] for p in orsted["places"]] == [ + "Storkøbenhavn", "Sydvestjylland (Esbjerg)", "Østjylland (Aarhus)"] + assert all("description" not in row and "teaser" not in row for row in _rows()) + + +def test_a_comma_in_the_employer_stays_with_the_employer() -> None: + row, = jobbank.rows(jobbank.page( + lambda url, **kw: _item("Fuldtidsjob hos AgainBio (Luabio Aps, Secondcircle Aps), " + "Storkøbenhavn (Ansøgningsfrist: løbende)"), + "", None, Ask())[0], "") + assert row["company"] == "AgainBio (Luabio Aps, Secondcircle Aps)" + assert [p["text"] for p in row["places"]] == ["Storkøbenhavn"] + + +def test_an_employer_with_no_region_after_it_keeps_its_whole_name() -> None: + row, = _rows(_item("Fuldtidsjob hos Acme, Inc. (Ansøgningsfrist: løbende)")) + assert row["company"] == "Acme, Inc." and row["places"] == [] + + +def test_the_country_is_stated_only_when_the_regions_name_one() -> None: + by_id = {row["source_id"]: row for row in _rows()} + assert by_id["3111914"]["country"] == "DK" + assert "country" not in by_id["3119220"] # Norway and "øvrige" + assert "country" not in by_id["3112442"] # Denmark, Sweden and Norway + assert _rows(_item("Fuldtidsjob hos Elopak, Udlandet (Norge) " + "(Ansøgningsfrist: løbende)"))[0]["country"] == "NO" + + +def test_regions_the_feed_cut_off_name_no_place_and_no_country() -> None: + cowi = next(row for row in _rows() if row["company"] == "COWI A/S") + assert [p["text"] for p in cowi["places"]] == ["Fyn", "Sydøstjylland", "Østjylland (Aarhus)"] + assert "country" not in cowi + row, = _rows(_item("Fuldtidsjob hos DANFOSS A/S, Storkøbenhavn +1 andre " + "(Ansøgningsfrist: løbende)")) + assert [p["text"] for p in row["places"]] == ["Storkøbenhavn"] and "country" not in row + + +def test_a_search_for_fully_remote_roles_marks_its_places_remote() -> None: + asked = Ask(params={"q": "x", "remote": "helt"}) + assert all(p["remote"] is True for row in _rows(REMOTE, asked) for p in row["places"]) + assert all(p["remote"] is None for row in _rows(REMOTE) for p in row["places"]) + partly = Ask(params={"q": "x", "remote": "delvist"}) + assert all(p["remote"] is None for row in _rows(REMOTE, partly) for p in row["places"]) + + +def test_it_asks_for_the_query_the_region_the_remote_word_and_the_page() -> None: + asked: list[str] = [] + + def get(url: str, **kwargs: Any) -> bytes: + asked.append(url) + return _feed(3) + + jobbank.page(get, "", None, Ask(params={"q": "data director", "region": "2", + "remote": "helt"})) + parts = urlsplit(asked[0]) + assert (parts.scheme, parts.netloc, parts.path) == ("https", "jobbank.dk", "/job/rss/") + assert parse_qs(parts.query) == {"key": ["data director"], "amt": ["2"], + "fjernarbejde": ["helt"], "page": ["1"]} + jobbank.page(get, "", 4, Ask(params={"q": "x"})) + assert "amt" not in asked[1] and "fjernarbejde" not in asked[1] and "page=4" in asked[1] + jobbank.page(get, "", None, Ask(query="y", places=("13",), params={"q": "x"})) + assert parse_qs(urlsplit(asked[2]).query)["key"] == ["y"] + assert parse_qs(urlsplit(asked[2]).query)["amt"] == ["13"] + + +@pytest.mark.parametrize("params", [{"q": "x", "region": "99"}, {"q": "x", "remote": "ja"}]) +def test_a_region_or_remote_word_it_does_not_know_is_not_requested(params: dict[str, str]) -> None: + def get(url: str, **kwargs: Any) -> bytes: + raise AssertionError("no request") + + with pytest.raises(ValueError, match="jobbank"): + jobbank.page(get, "", None, Ask(params=params)) + + +@pytest.mark.parametrize(("items", "cursor", "after"), [ + (100, None, 2), # a full page has a next + (99, None, None), # a short page is the end + (0, None, None), + (100, 20, None), # the twentieth page reaches the 2,000 ceiling + (100, 19, 20), +]) +def test_it_pages_until_a_short_page_or_the_ceiling( + items: int, cursor: int | None, after: int | None) -> None: + _decoded, results, total, nxt = jobbank.page( + lambda url, **kw: _feed(items), "", cursor, Ask(params={"q": "x"})) + assert (len(results), total, nxt) == (items, None, after) + + +def test_an_empty_page_ends_the_list() -> None: + decoded, results, total, nxt = jobbank.page( + lambda url, **kw: EMPTY, "", 17, Ask(params={"q": "x"})) + assert (results, total, nxt, jobbank.rows(decoded, "")) == ([], None, None, []) + + +def test_a_dropped_item_does_not_pass_for_the_end() -> None: + body = _feed(99).replace(b"", b"No link") + decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask()) + assert len(results) == 100 and nxt == 2 + assert len(jobbank.rows(decoded, "")) == 99 + + +@pytest.mark.parametrize("body", [b"", b"Just a moment...", + b"Just a moment...

"]) +def test_an_answer_that_is_not_a_feed_raises(body: bytes) -> None: + with pytest.raises(ValueError, match="jobbank"): + jobbank.page(lambda url, **kw: body, "", None, Ask()) + + +def test_saved_params_keep_the_query_a_real_region_and_a_remote_word() -> None: + assert jobbank.clean_params({"q": "x"}) == {"q": "x"} + assert jobbank.clean_params({"q": "x", "region": 2, "remote": "delvist"}) == { + "q": "x", "region": "2", "remote": "delvist"} + assert jobbank.clean_params({"q": "x", "region": "", "remote": ""}) == {"q": "x"} + with pytest.raises(ValueError, match="region"): + jobbank.clean_params({"q": "x", "region": "99"}) + with pytest.raises(ValueError, match="remote"): + jobbank.clean_params({"q": "x", "remote": "true"}) + + +# ------------------------------------------------------------------ detail + +def _detail(row: dict[str, Any] | None = None, body: bytes = AD) -> dict[str, Any]: + row = row or _rows()[0] + return jobbank.detail(lambda url, **kw: body, "", row) # type: ignore[arg-type] + + +def test_detail_takes_the_description_and_adds_the_town_to_the_regions() -> None: + found = _detail() + assert found["description"].startswith("Vil du være med til at opbygge Historisk Personregister") + assert "## Om Rigsarkivet" in found["description"] and "æ" not in found["description"] + assert [p["text"] for p in found["places"]] == ["Storkøbenhavn", "Fyn", "Odense"] + assert "posted" not in found # the feed has the date + + +def test_detail_dates_a_role_the_feed_did_not() -> None: + row = _rows()[0] | {"posted": None} + assert _detail(row)["posted"].date().isoformat() == "2026-09-04" + + +def test_detail_asks_for_the_ads_own_page_only() -> None: + asked: list[str] = [] + + def get(url: str, **kwargs: Any) -> bytes: + asked.append(url) + return AD + + row = _rows()[0] + jobbank.detail(get, "", row) # type: ignore[arg-type] + assert asked == [row["url"]] + + +@pytest.mark.parametrize("url", ["https://jobbank.dk/fwd/webjob/3111914/", + "https://jobbank.dk/link/3111914", + "https://example.com/job/3111914/x"]) +def test_detail_never_follows_anything_but_a_job_page(url: str) -> None: + def get(url: str, **kwargs: Any) -> bytes: + raise AssertionError("no request") + + with pytest.raises(ValueError, match="/job/"): + jobbank.detail(get, "", _rows()[0] | {"url": url}) # type: ignore[arg-type] + + +@pytest.mark.parametrize("body", [ + b"Just a moment...Checking your browser", + b'', + b'', + b""]) +def test_a_page_without_a_job_posting_is_an_error_not_an_empty_text(body: bytes) -> None: + with pytest.raises(ValueError, match="JobPosting"): + _detail(body=body) + + +def test_detail_finds_the_posting_among_other_blocks_and_in_a_list() -> None: + posting = '{"@type": "JobPosting", "description": "

Hej

"}' + body = ('' + f'').encode() + assert _detail(body=body)["description"] == "Hej" + + +def test_detail_without_a_town_leaves_the_places_as_they_are() -> None: + body = b'' + assert "places" not in _detail(body=body)