Merge the jobnet search reader (wp-src-jobnet)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-10-03 12:21:18 +02:00
commit ee17fe3e30
5 files changed
+545

No files matched your search

+160
View File
@@ -0,0 +1,160 @@
"""Jobnet: Denmark's state job portal (jobnet.dk), searched through the
backend of its own site, `/bff/FindJob/Search`; no key, no login.
Checked live on 2026-10-03. The endpoint answers 401 without the header
`x-csrf: 1` and 200 with it, whatever its value; every request sends it
through `get`. jobnet.dk has no robots.txt: `/robots.txt` redirects to a login
page. `regions` is one of the five region ids in `REGIONS`, empty for all of
Denmark. 100 results a page (`resultsPerPage`), `pageNumber` from 1, ordered
by `PublicationDate`, and the page after the 100th answers an empty list, so
a search reads 10,000 results at most. The answer states `totalJobAdCount`,
and every ad has its whole text as HTML, so the list has the full text.
`searchString` is a loose match with no phrase syntax: a quoted phrase finds
nothing.
Mostly Danish public sector and small employers; about a fifth of the ads
only link to the employer's own page (`jobAdUrl`), which is then the role's
`url`; the rest open Jobnet's own page for the ad. `country` is written in
Danish ("Tyskland"), and a few ads are abroad. An ad states one place, the
municipality or else the postal district, and none when its address is
blank. `applicationDeadline`, with `1900-01-01` for none, is kept in `raw`.
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import code_of
from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt, place
NAME = "jobnet"
LABEL = "Jobnet (Denmark)"
RANK = 40
READS = Reads(
scope="search", listing="paged", page_size=100, ceiling=10000,
text="listing", complete="stated",
filters=Filters(query="one", place="one"),
places="one",
fields=Fields(posted="trusted", country="stated"),
robots="read: jobnet.dk has no robots.txt, /robots.txt redirects to a login; "
"/bff/FindJob/Search answers 401 without the header x-csrf: 1 and "
"200 with it (checked 2026-10-03)")
#: Saved param -> the filter dimension it fills.
PARAMS = {"q": "query", "region": "place"}
SEARCH = "https://jobnet.dk/bff/FindJob/Search"
#: What the backend wants on every request; any value does.
HEADERS = {"x-csrf": "1"}
#: The five regions as the `regions` facet of an answer names them, with the
#: label the site shows. "" is all of Denmark and is not a key here.
REGIONS: dict[str, str] = {
"HovedstadenOgBornholm": "Hovedstaden og Bornholm",
"Midtjylland": "Midtjylland",
"Syddanmark": "Syddanmark",
"OevrigeSjaelland": "Øvrige Sjælland",
"Nordjylland": "Nordjylland",
}
#: The countries of the `countries` facet (2026-10-03), as the ads write them
#: in Danish, to ISO codes; a name not here is read as an English one.
COUNTRIES: dict[str, str] = {
"Danmark": "DK", "Grønland": "GL", "Grækenland": "GR", "Norge": "NO",
"Tyskland": "DE", "Spanien": "ES", "Sverige": "SE", "Malta": "MT", "Østrig": "AT",
"Amerikas Forenede Stater (USA)": "US", "Cypern": "CY", "Portugal": "PT",
"Bulgarien": "BG", "Schweiz": "CH", "Marokko": "MA", "Nederlandene": "NL",
"Vietnam": "VN", "Canada": "CA", "Tjekkiet": "CZ", "Finland": "FI", "Færøerne": "FO",
"Indien": "IN", "Malaysia": "MY", "Filippinerne": "PH", "Polen": "PL",
"Saudi Arabien": "SA", "Singapore": "SG", "Storbritannien": "GB", "Frankrig": "FR",
"Italien": "IT", "Irland": "IE", "Belgien": "BE", "Island": "IS", "Estland": "EE",
"Letland": "LV", "Litauen": "LT", "Ungarn": "HU", "Rumænien": "RO", "Kroatien": "HR",
"Tyrkiet": "TR", "Kina": "CN", "Japan": "JP", "Australien": "AU", "Luxembourg": "LU",
}
#: `applicationDeadline` of an ad with no deadline.
_NO_DEADLINE = "1900-01-01"
def allowed(slug: str) -> Callable[[str], bool]:
return only("jobnet.dk")
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
"""The saved params of a search: `q` and `region`, the id of one of the
five regions or empty for all of Denmark."""
region = str(params.get("region") or "").strip()
if region and region not in REGIONS:
raise ValueError("region is empty or one of " + ", ".join(REGIONS))
return {"q": params["q"], "region": region}
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search, newest first; the cursor is the page number,
from 1. The next page follows a full page while it is within the
ceiling. An answer without `jobAds` is not a search result and raises,
since an empty list would read as a search that ran out."""
region = str(ask.places[0] if ask.places else ask.params.get("region") or "")
if region and region not in REGIONS:
raise ValueError(f"jobnet region {region!r} is not one of the ids in REGIONS")
size, ceiling = READS.page_size or 100, READS.ceiling or 10000
number = int(cursor or 1)
query: dict[str, Any] = {
"searchString": ask.query or str(ask.params.get("q") or ""),
"orderType": "PublicationDate", "resultsPerPage": size, "pageNumber": number}
if region:
query["regions"] = region
data = json.loads(get(SEARCH + "?" + urlencode(query), headers=HEADERS))
if not isinstance(data, dict) or not isinstance(data.get("jobAds"), list):
raise ValueError("jobnet: the answer has no jobAds")
ads = [ad for ad in data["jobAds"] if isinstance(ad, dict)]
total = data.get("totalJobAdCount")
more = len(ads) >= size and (number + 1) * size <= ceiling
return (data, [{"id": str(ad.get("jobAdId") or "")} for ad in ads],
total if isinstance(total, int) else None, number + 1 if more else None)
def _text(fragment: Any) -> str:
return strip_html(str(fragment or "")).strip()
def _country(name: str) -> str | None:
"""The ISO code of an ad's country, `DK` when it names none."""
name = name.strip()
return (COUNTRIES.get(name) or code_of(name) or None) if name else "DK"
def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = []
for ad in decoded.get("jobAds") or []:
ident = str(ad.get("jobAdId") or "").strip()
title = _text(ad.get("title"))
if not ident or not title:
continue
link = str(ad.get("jobAdUrl") or "").strip()
town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
places: list[Place] = [place(town)] if town else []
row: Row = {
"source_id": ident,
"title": title,
"company": _text(ad.get("hiringOrgName")),
"url": link if is_web_url(link) else f"https://jobnet.dk/find-job/{ident}",
"places": places,
"location": town,
"country": _country(str(ad.get("country") or "")),
"posted": parse_dt(ad.get("publicationDate")),
"description": html_text(ad.get("description")),
}
deadline = str(ad.get("applicationDeadline") or "")
if deadline and not deadline.startswith(_NO_DEADLINE):
row["applicationDeadline"] = deadline
out.append(row)
return out