Add the Jobnet search reader, which reads jobnet.dk's backend with the x-csrf header and 100 ads a page
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
018a1ad895
commit
0ed0360b09
5 files changed
+545
No files matched your search
@@ -0,0 +1,160 @@
|
||||
"""Jobnet: Denmark's state job portal (jobnet.dk), searched through the
|
||||
backend of its own site, `/bff/FindJob/Search`; no key, no login.
|
||||
|
||||
Checked live on 2026-10-03. The endpoint answers 401 without the header
|
||||
`x-csrf: 1` and 200 with it, whatever its value; every request sends it
|
||||
through `get`. jobnet.dk has no robots.txt: `/robots.txt` redirects to a login
|
||||
page. `regions` is one of the five region ids in `REGIONS`, empty for all of
|
||||
Denmark. 100 results a page (`resultsPerPage`), `pageNumber` from 1, ordered
|
||||
by `PublicationDate`, and the page after the 100th answers an empty list, so
|
||||
a search reads 10,000 results at most. The answer states `totalJobAdCount`,
|
||||
and every ad has its whole text as HTML, so the list has the full text.
|
||||
`searchString` is a loose match with no phrase syntax: a quoted phrase finds
|
||||
nothing.
|
||||
|
||||
Mostly Danish public sector and small employers; about a fifth of the ads
|
||||
only link to the employer's own page (`jobAdUrl`), which is then the role's
|
||||
`url`; the rest open Jobnet's own page for the ad. `country` is written in
|
||||
Danish ("Tyskland"), and a few ads are abroad. An ad states one place, the
|
||||
municipality or else the postal district, and none when its address is
|
||||
blank. `applicationDeadline`, with `1900-01-01` for none, is kept in `raw`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import code_of
|
||||
from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt, place
|
||||
|
||||
NAME = "jobnet"
|
||||
LABEL = "Jobnet (Denmark)"
|
||||
RANK = 40
|
||||
|
||||
READS = Reads(
|
||||
scope="search", listing="paged", page_size=100, ceiling=10000,
|
||||
text="listing", complete="stated",
|
||||
filters=Filters(query="one", place="one"),
|
||||
places="one",
|
||||
fields=Fields(posted="trusted", country="stated"),
|
||||
robots="read: jobnet.dk has no robots.txt, /robots.txt redirects to a login; "
|
||||
"/bff/FindJob/Search answers 401 without the header x-csrf: 1 and "
|
||||
"200 with it (checked 2026-10-03)")
|
||||
|
||||
#: Saved param -> the filter dimension it fills.
|
||||
PARAMS = {"q": "query", "region": "place"}
|
||||
|
||||
SEARCH = "https://jobnet.dk/bff/FindJob/Search"
|
||||
|
||||
#: What the backend wants on every request; any value does.
|
||||
HEADERS = {"x-csrf": "1"}
|
||||
|
||||
#: The five regions as the `regions` facet of an answer names them, with the
|
||||
#: label the site shows. "" is all of Denmark and is not a key here.
|
||||
REGIONS: dict[str, str] = {
|
||||
"HovedstadenOgBornholm": "Hovedstaden og Bornholm",
|
||||
"Midtjylland": "Midtjylland",
|
||||
"Syddanmark": "Syddanmark",
|
||||
"OevrigeSjaelland": "Øvrige Sjælland",
|
||||
"Nordjylland": "Nordjylland",
|
||||
}
|
||||
|
||||
#: The countries of the `countries` facet (2026-10-03), as the ads write them
|
||||
#: in Danish, to ISO codes; a name not here is read as an English one.
|
||||
COUNTRIES: dict[str, str] = {
|
||||
"Danmark": "DK", "Grønland": "GL", "Grækenland": "GR", "Norge": "NO",
|
||||
"Tyskland": "DE", "Spanien": "ES", "Sverige": "SE", "Malta": "MT", "Østrig": "AT",
|
||||
"Amerikas Forenede Stater (USA)": "US", "Cypern": "CY", "Portugal": "PT",
|
||||
"Bulgarien": "BG", "Schweiz": "CH", "Marokko": "MA", "Nederlandene": "NL",
|
||||
"Vietnam": "VN", "Canada": "CA", "Tjekkiet": "CZ", "Finland": "FI", "Færøerne": "FO",
|
||||
"Indien": "IN", "Malaysia": "MY", "Filippinerne": "PH", "Polen": "PL",
|
||||
"Saudi Arabien": "SA", "Singapore": "SG", "Storbritannien": "GB", "Frankrig": "FR",
|
||||
"Italien": "IT", "Irland": "IE", "Belgien": "BE", "Island": "IS", "Estland": "EE",
|
||||
"Letland": "LV", "Litauen": "LT", "Ungarn": "HU", "Rumænien": "RO", "Kroatien": "HR",
|
||||
"Tyrkiet": "TR", "Kina": "CN", "Japan": "JP", "Australien": "AU", "Luxembourg": "LU",
|
||||
}
|
||||
|
||||
#: `applicationDeadline` of an ad with no deadline.
|
||||
_NO_DEADLINE = "1900-01-01"
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only("jobnet.dk")
|
||||
|
||||
|
||||
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The saved params of a search: `q` and `region`, the id of one of the
|
||||
five regions or empty for all of Denmark."""
|
||||
region = str(params.get("region") or "").strip()
|
||||
if region and region not in REGIONS:
|
||||
raise ValueError("region is empty or one of " + ", ".join(REGIONS))
|
||||
return {"q": params["q"], "region": region}
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of a search, newest first; the cursor is the page number,
|
||||
from 1. The next page follows a full page while it is within the
|
||||
ceiling. An answer without `jobAds` is not a search result and raises,
|
||||
since an empty list would read as a search that ran out."""
|
||||
region = str(ask.places[0] if ask.places else ask.params.get("region") or "")
|
||||
if region and region not in REGIONS:
|
||||
raise ValueError(f"jobnet region {region!r} is not one of the ids in REGIONS")
|
||||
size, ceiling = READS.page_size or 100, READS.ceiling or 10000
|
||||
number = int(cursor or 1)
|
||||
query: dict[str, Any] = {
|
||||
"searchString": ask.query or str(ask.params.get("q") or ""),
|
||||
"orderType": "PublicationDate", "resultsPerPage": size, "pageNumber": number}
|
||||
if region:
|
||||
query["regions"] = region
|
||||
data = json.loads(get(SEARCH + "?" + urlencode(query), headers=HEADERS))
|
||||
if not isinstance(data, dict) or not isinstance(data.get("jobAds"), list):
|
||||
raise ValueError("jobnet: the answer has no jobAds")
|
||||
ads = [ad for ad in data["jobAds"] if isinstance(ad, dict)]
|
||||
total = data.get("totalJobAdCount")
|
||||
more = len(ads) >= size and (number + 1) * size <= ceiling
|
||||
return (data, [{"id": str(ad.get("jobAdId") or "")} for ad in ads],
|
||||
total if isinstance(total, int) else None, number + 1 if more else None)
|
||||
|
||||
|
||||
def _text(fragment: Any) -> str:
|
||||
return strip_html(str(fragment or "")).strip()
|
||||
|
||||
|
||||
def _country(name: str) -> str | None:
|
||||
"""The ISO code of an ad's country, `DK` when it names none."""
|
||||
name = name.strip()
|
||||
return (COUNTRIES.get(name) or code_of(name) or None) if name else "DK"
|
||||
|
||||
|
||||
def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
out: list[Row] = []
|
||||
for ad in decoded.get("jobAds") or []:
|
||||
ident = str(ad.get("jobAdId") or "").strip()
|
||||
title = _text(ad.get("title"))
|
||||
if not ident or not title:
|
||||
continue
|
||||
link = str(ad.get("jobAdUrl") or "").strip()
|
||||
town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
|
||||
places: list[Place] = [place(town)] if town else []
|
||||
row: Row = {
|
||||
"source_id": ident,
|
||||
"title": title,
|
||||
"company": _text(ad.get("hiringOrgName")),
|
||||
"url": link if is_web_url(link) else f"https://jobnet.dk/find-job/{ident}",
|
||||
"places": places,
|
||||
"location": town,
|
||||
"country": _country(str(ad.get("country") or "")),
|
||||
"posted": parse_dt(ad.get("publicationDate")),
|
||||
"description": html_text(ad.get("description")),
|
||||
}
|
||||
deadline = str(ad.get("applicationDeadline") or "")
|
||||
if deadline and not deadline.startswith(_NO_DEADLINE):
|
||||
row["applicationDeadline"] = deadline
|
||||
out.append(row)
|
||||
return out
|
||||
Reference in new issue
Block a user