Read Akademikernes Jobbank's search feed, with each ad's text from its own page
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
018a1ad895
commit
43c0bc7523
6 files changed
+670
No files matched your search
@@ -0,0 +1,253 @@
|
||||
"""Akademikernes Jobbank (jobbank.dk): Denmark's portal for academics and
|
||||
researchers, with corporate ads (LEO Pharma, Ørsted) beside the public sector,
|
||||
searched through the RSS feed of its search, `/job/rss/`; no key.
|
||||
|
||||
Checked live on 2026-10-03. The trailing slash matters, `/job/rss` answers 301.
|
||||
`key` is the query, words are OR. `amt` is the region of the site's own search
|
||||
form (2 is Storkøbenhavn), `fjernarbejde` is `helt` or `delvist`. 100 items a
|
||||
page and an empty channel ends the list. The feed states no total and is not
|
||||
sorted by date, so a search is never complete and the newest ads may be on any
|
||||
page.
|
||||
|
||||
An item is `<title>` the role, a `<link>` to the ad's page on jobbank.dk and a
|
||||
`<description>` of one line: "Fuldtidsjob hos LEO PHARMA A/S, Storkøbenhavn,
|
||||
Fyn (Ansøgningsfrist: 18.10.2026)", the kind of job, the employer, up to three
|
||||
of the regions the ad is for (the rest as "+12 andre", which names none) and
|
||||
the deadline or "løbende". The employer may hold a comma, so the regions are
|
||||
taken from the end, as far as they are regions of the form. The feed has no
|
||||
text: `detail` reads the ad's own page, whose JSON-LD `JobPosting` has the
|
||||
description, the town and the dates. The page's "Ansøg" button goes through
|
||||
`/fwd/webjob/`, which robots.txt disallows and which is never requested; a
|
||||
Cloudflare challenge in place of the page is an error for that role, not an
|
||||
empty text.
|
||||
|
||||
`fjernarbejde` is a word, not a flag, so the saved param stays in `Ask.params`
|
||||
(the runner takes a bool only for the `remote` dimension) and `page` reads it
|
||||
there. A search asked for `helt` lists remote roles only, and its places say so.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt, place
|
||||
|
||||
NAME = "jobbank"
|
||||
LABEL = "Akademikernes Jobbank (Denmark)"
|
||||
|
||||
READS = Reads(
|
||||
scope="search", listing="paged", page_size=100, ceiling=2000,
|
||||
text="detail", complete="never",
|
||||
filters=Filters(query="one", place="one", remote="one"),
|
||||
places="many",
|
||||
fields=Fields(posted="trusted", country="stated"),
|
||||
robots="read: robots.txt allows /job/ and disallows /fwd/webjob/, /link/, /redirect/ "
|
||||
"and /ajax/ with Host-load 2; the reader requests /job/rss/ and /job/<id>/ only "
|
||||
"and never follows /fwd/webjob/ (checked 2026-10-03)")
|
||||
|
||||
#: Saved param -> the filter dimension it fills.
|
||||
PARAMS = {"q": "query", "region": "place", "remote": "remote"}
|
||||
|
||||
RSS = "https://jobbank.dk/job/rss/"
|
||||
|
||||
#: The `amt` ids of the site's own search form, with its labels. Empty is all of
|
||||
#: Denmark and not a key here. "Øresundsregionen" and the "Udlandet" ids are
|
||||
#: places beyond Denmark's own regions.
|
||||
REGIONS: dict[str, str] = {
|
||||
"2": "Storkøbenhavn", "3": "Nordsjælland", "14": "Østsjælland", "4": "Vestsjælland",
|
||||
"5": "Sydsjælland & Øer", "13": "Fyn", "12": "Sønderjylland",
|
||||
"11": "Sydvestjylland (Esbjerg)", "9": "Vestjylland", "10": "Sydøstjylland",
|
||||
"7": "Midtjylland", "8": "Østjylland (Aarhus)", "6": "Nordjylland", "20": "Bornholm",
|
||||
"21": "Øresundsregionen", "22": "Grønland & Færøerne", "23": "Udlandet (Sverige)",
|
||||
"24": "Udlandet (Norge)", "19": "Udlandet (øvrige)",
|
||||
}
|
||||
|
||||
#: The `fjernarbejde` values of the form: fully and partly remote.
|
||||
REMOTE: dict[str, str] = {"helt": "Fully remote", "delvist": "Partly remote"}
|
||||
|
||||
_LABELS = frozenset(REGIONS.values())
|
||||
|
||||
#: The country a region lies in, where it lies in one. A role with every listed
|
||||
#: region in one country has that country; "Øresundsregionen", "Grønland &
|
||||
#: Færøerne" and "Udlandet (øvrige)" name none.
|
||||
_COUNTRY: dict[str, str] = {
|
||||
label: "DK" for label in REGIONS.values()
|
||||
if label not in ("Øresundsregionen", "Grønland & Færøerne") and not label.startswith("Udlandet")
|
||||
} | {"Udlandet (Sverige)": "SE", "Udlandet (Norge)": "NO"}
|
||||
|
||||
_ID = re.compile(r"/job/(\d+)(?:/|$)")
|
||||
#: "Fuldtidsjob, Graduate/trainee hos LEO PHARMA A/S, Storkøbenhavn +2 andre
|
||||
#: (Ansøgningsfrist: 18.10.2026)": the kind, the employer and regions, the deadline.
|
||||
_LINE = re.compile(r"(?P<kind>.*?) hos (?P<rest>.*?)\s*"
|
||||
r"(?:\(Ansøgningsfrist:\s*(?P<deadline>[^)]*)\))?\s*", re.DOTALL)
|
||||
_MORE = re.compile(r"\s*\+\d+ andre$")
|
||||
_DEADLINE = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})")
|
||||
_JSON_LD = re.compile(r"<script[^>]*application/ld\+json[^>]*>(.*?)</script>",
|
||||
re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only("jobbank.dk")
|
||||
|
||||
|
||||
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The saved params of a search: `q`, `region` when it is the id of a
|
||||
region of the form and `remote` when it is `helt` or `delvist`."""
|
||||
out: dict[str, Any] = {"q": params["q"]}
|
||||
region = str(params.get("region") or "").strip()
|
||||
if region:
|
||||
if region not in REGIONS:
|
||||
raise ValueError("region is empty or the id of a region of jobbank.dk, "
|
||||
"2 is Storkøbenhavn")
|
||||
out["region"] = region
|
||||
remote = str(params.get("remote") or "").strip()
|
||||
if remote:
|
||||
if remote not in REMOTE:
|
||||
raise ValueError("remote is empty, helt or delvist")
|
||||
out["remote"] = remote
|
||||
return out
|
||||
|
||||
|
||||
def _query(ask: Ask, number: int) -> tuple[str, str]:
|
||||
"""The feed's URL for page `number`, and the `fjernarbejde` it asks for."""
|
||||
region = str(ask.places[0] if ask.places else ask.params.get("region") or "")
|
||||
remote = str(ask.params.get("remote") or "")
|
||||
if region and region not in REGIONS:
|
||||
raise ValueError(f"jobbank region {region!r} is not one of the ids in REGIONS")
|
||||
if remote and remote not in REMOTE:
|
||||
raise ValueError(f"jobbank remote {remote!r} is not one of the words in REMOTE")
|
||||
query: dict[str, Any] = {"key": ask.query or str(ask.params.get("q") or "")}
|
||||
if region:
|
||||
query["amt"] = region
|
||||
if remote:
|
||||
query["fjernarbejde"] = remote
|
||||
return RSS + "?" + urlencode(query | {"page": number}), remote
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of a search; the cursor is the page number. The next page
|
||||
follows a full page (counted before any item is dropped, so a dropped item
|
||||
does not pass for the end) while the ceiling is not reached."""
|
||||
number = int(cursor or 1)
|
||||
url, remote = _query(ask, number)
|
||||
try:
|
||||
root = ET.fromstring(get(url))
|
||||
except ET.ParseError as exc:
|
||||
raise ValueError(f"jobbank answered something that is not RSS ({exc}), "
|
||||
"a challenge in place of the feed?") from exc
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
raise ValueError("jobbank answered without an RSS channel")
|
||||
items = []
|
||||
for item in channel.findall("item"):
|
||||
link = (item.findtext("link") or "").strip()
|
||||
found = _ID.search(link)
|
||||
items.append({
|
||||
"id": found.group(1) if found else "",
|
||||
"link": link,
|
||||
"title": (item.findtext("title") or "").strip(),
|
||||
"description": (item.findtext("description") or "").strip(),
|
||||
"published": item.findtext("pubDate"),
|
||||
})
|
||||
decoded = {"remote": remote, "items": items}
|
||||
size, ceiling = READS.page_size or 100, READS.ceiling or 2000
|
||||
more = len(items) >= size and number * size < ceiling
|
||||
return decoded, [{"id": item["id"]} for item in items], None, number + 1 if more else None
|
||||
|
||||
|
||||
def _regions(rest: str) -> tuple[str, list[str], bool]:
|
||||
"""`(company, regions, cut)` of what follows "hos": the regions are the
|
||||
known labels at the end, "+12 andre" (`cut`) the ones the feed leaves out.
|
||||
Nothing but the employer is left when no label is at the end."""
|
||||
cut = bool(_MORE.search(rest))
|
||||
parts = _MORE.sub("", rest).split(", ")
|
||||
regions: list[str] = []
|
||||
while len(parts) > 1 and parts[-1].strip() in _LABELS:
|
||||
regions.insert(0, parts.pop().strip())
|
||||
return ", ".join(parts).strip(), regions, cut
|
||||
|
||||
|
||||
def _country(regions: list[str], cut: bool) -> str | None:
|
||||
"""The one country all the listed regions lie in; None when they lie in
|
||||
several or in none, or when "+N andre" left some out."""
|
||||
found = {_COUNTRY.get(region) for region in regions}
|
||||
return found.pop() if len(found) == 1 and not cut and None not in found else None
|
||||
|
||||
|
||||
def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
out: list[Row] = []
|
||||
for item in decoded["items"]:
|
||||
link, title = item["link"], item["title"]
|
||||
if not title or not item["id"] or not is_web_url(link):
|
||||
continue
|
||||
line = _LINE.fullmatch(item["description"])
|
||||
kind = line.group("kind").strip() if line else ""
|
||||
company, regions, cut = _regions(line.group("rest")) if line else ("", [], False)
|
||||
places: list[Place] = [place(region, True if decoded["remote"] == "helt" else None)
|
||||
for region in regions]
|
||||
row: Row = {
|
||||
"source_id": item["id"],
|
||||
"title": title,
|
||||
"company": company,
|
||||
"url": link,
|
||||
"places": places,
|
||||
"location": regions[0] if regions else "",
|
||||
"posted": parse_dt(item["published"]),
|
||||
}
|
||||
if country := _country(regions, cut):
|
||||
row["country"] = country
|
||||
if kind:
|
||||
row["jobType"] = kind
|
||||
if line and (day := _DEADLINE.fullmatch((line.group("deadline") or "").strip())):
|
||||
row["applicationDeadline"] = f"{day.group(3)}-{day.group(2)}-{day.group(1)}"
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
|
||||
def _posting(body: bytes) -> dict[str, Any]:
|
||||
"""The JSON-LD `JobPosting` of an ad's page. Raises ValueError when the
|
||||
page has none, as a Cloudflare challenge page has not."""
|
||||
for found in _JSON_LD.finditer(body.decode("utf-8", errors="replace")):
|
||||
try:
|
||||
data = json.loads(found.group(1), strict=False)
|
||||
except ValueError:
|
||||
continue
|
||||
for node in data if isinstance(data, list) else [data]:
|
||||
if isinstance(node, dict) and "JobPosting" in (
|
||||
node["@type"] if isinstance(node.get("@type"), list) else [node.get("@type")]):
|
||||
return node
|
||||
raise ValueError("jobbank answered a page without a JobPosting, a challenge in place of the ad?")
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""The ad's own page: the description, the town added to the places the
|
||||
feed named, and the date when the feed gave none. Only a `/job/` page of
|
||||
jobbank.dk is requested, never `/fwd/webjob/`. `validThrough` is not
|
||||
taken: the feed's deadline is already in `raw`, and `detail` may fill
|
||||
`_runner.DETAIL_FIELDS` only."""
|
||||
url = row["url"]
|
||||
parts = urlsplit(url)
|
||||
if parts.hostname != "jobbank.dk" or not parts.path.startswith("/job/"):
|
||||
raise ValueError(f"jobbank reads /job/ pages of jobbank.dk only, not {url}")
|
||||
posting = _posting(get(url))
|
||||
out: dict[str, Any] = {}
|
||||
if description := html_text(posting.get("description")):
|
||||
out["description"] = description
|
||||
location = posting.get("jobLocation")
|
||||
if isinstance(location, list):
|
||||
location = location[0] if location else None
|
||||
address = location.get("address") if isinstance(location, dict) else None
|
||||
town = str(address.get("addressLocality") or "").strip() if isinstance(address, dict) else ""
|
||||
if town:
|
||||
found: list[Place] = [dict(p) for p in row.get("places") or []] # type: ignore[misc]
|
||||
add_place(found, place(town))
|
||||
out["places"] = found
|
||||
if not row.get("posted") and (posted := parse_dt(posting.get("datePosted"))):
|
||||
out["posted"] = posted
|
||||
return out
|
||||
Reference in new issue
Block a user