Search providers as reader modules: jobindex, adzuna, platsbanken

Each declares READS with scope=search, keeps every place and hands the excerpt over as teaser. search_params and PROVIDERS follow the declarations.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:08:25 +02:00
1 parent 4c0da58da7
commit b78c9dfc67
14 files changed
+1202 -17

No files matched your search

+19
View File
@@ -0,0 +1,19 @@
"""What the search providers' headlines and town lines say about where the
work is done, in Danish, Swedish and English. Shared by the search readers;
a board states its own remote flag and needs none of this."""
from __future__ import annotations
import re
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def remote_word(text: str) -> str | None:
"""`remote`, `hybrid` or None: the word the text says."""
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
+135
View File
@@ -0,0 +1,135 @@
"""Adzuna: an aggregator with a documented API for most of Europe, the UK and
the US, but not Denmark (adzuna.dk does not exist). Needs a free key from
developer.adzuna.com, as `sources.search.adzuna: {app_id, app_key}` or
`ADZUNA_APP_ID` and `ADZUNA_APP_KEY`; the runner resolves it and passes it as
`ask.params["key"]`.
Built against Adzuna's documented response and not yet run live. One request
is `/v1/api/jobs/<country>/search/<page>` with `what`, `where`, `sort_by=date`
and 50 results a page; the answer is `{"count", "results": [...]}` and the
search is complete when as many distinct results were read as `count` states.
Titles and snippets carry `<strong>` highlights. The description is a snippet
of the ad, so it is the `teaser`. A salary Adzuna estimated
(`salary_is_predicted`) is not the posting's, so it is dropped.
"""
from __future__ import annotations
import html
import json
from collections.abc import Callable
from typing import Any
from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Key, Place, Reads
from ._words import remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "adzuna"
LABEL = "Adzuna"
RANK = 20
READS = Reads(
scope="search", listing="paged", page_size=50,
text="excerpt", complete="stated",
filters=Filters(query="one", country="one", place="one"),
places="one",
fields=Fields(pay="figures", posted="trusted", country="stated"),
key=Key(name="adzuna", fields=("app_id", "app_key"),
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")),
robots="robots.txt disallows / for every agent (checked 2026-09-30); the documented "
"API is read with the account's key, decision of the owner pending")
#: Saved param -> the filter dimension it fills.
PARAMS = {"q": "query", "country": "country", "where": "place"}
SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
#: Adzuna's countries in Europe, and the currency its salaries are in.
COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
def allowed(slug: str) -> Callable[[str], bool]:
return only("api.adzuna.com")
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
"""The saved params of a search: `q`, `country` (one of `COUNTRIES`) and
`where` when given."""
country = str(params.get("country") or "").strip().lower()
if country not in COUNTRIES:
raise ValueError(f"country is one of {', '.join(sorted(COUNTRIES))}")
out: dict[str, Any] = {"q": params["q"], "country": country}
if str(params.get("where") or "").strip():
out["where"] = str(params["where"]).strip()
return out
def _text(fragment: str) -> str:
return html.unescape(strip_html(fragment)).strip()
def _country(ask: Ask) -> str:
country = (ask.countries[0] if ask.countries else str(ask.params.get("country") or ""))
country = country.lower()
if country and country not in COUNTRIES:
raise ValueError(f"adzuna country {country!r} is not one of "
f"{', '.join(sorted(COUNTRIES))}")
return country
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search, newest first; the cursor is the page number.
The next page follows a full page. A missing key or country is not
refused here: the runner resolves the key (`READS.key`) and refuses a
search without one, `clean_params` refuses a search without a country,
and Adzuna answers 401 or 404 to a request that lacks either."""
key = ask.params.get("key") or {}
country = _country(ask)
number = int(cursor or 1)
query: dict[str, Any] = {"app_id": key.get("app_id", ""), "app_key": key.get("app_key", ""),
"results_per_page": READS.page_size, "sort_by": "date",
"what": ask.query or str(ask.params.get("q") or "")}
where = ask.places[0] if ask.places else ask.params.get("where")
if where:
query["where"] = where
url = SEARCH.format(country=country, page=number) + "?" + urlencode(query)
data = json.loads(get(url))
results = [job for job in data.get("results") or [] if isinstance(job, dict)]
count = data.get("count")
decoded = {"country": country, "results": results}
more = len(results) >= (READS.page_size or 50)
return (decoded, [{"id": str(job.get("id") or "")} for job in results],
count if isinstance(count, int) else None, number + 1 if more else None)
def rows(decoded: Any, slug: str) -> list[Row]:
country = decoded["country"]
out: list[Row] = []
for job in decoded["results"]:
title = _text(job.get("title") or "")
url = (job.get("redirect_url") or "").strip()
if not title or not is_web_url(url):
continue
predicted = str(job.get("salary_is_predicted") or "0") == "1"
town = ((job.get("location") or {}).get("display_name") or "").strip()
places: list[Place] = [{"text": town, "remote": None}] if town else []
out.append({
"source_id": str(job.get("id") or ""),
"title": title,
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
"url": url,
"places": places,
"location": town,
"country": country.upper(),
"remote": remote_word(title),
"posted": parse_dt(job.get("created")),
"teaser": _text(job.get("description") or ""),
"salary_min": None if predicted else job.get("salary_min"),
"salary_max": None if predicted else job.get("salary_max"),
"currency": None if predicted else COUNTRIES.get(country),
})
return out
+150
View File
@@ -0,0 +1,150 @@
"""Jobindex: Denmark's largest job board, searched through the RSS twin of
its search page, `/jobsoegning.rss`; no key.
Checked live on 2026-09-23 and moved here from `plugins/search.py`. `q` takes
Jobindex's own syntax: words are OR, `"a b"` a phrase, `+word` required.
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region Sjælland, 2
Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. 20 results a page, and no
page past the tenth answers anything, so a search reads 200 results at most.
The feed states no total, so a search is never complete.
An item is `<title>` "Role, Company", a `<link>` to Jobindex's own page for it
and a `<description>` of HTML: the `<h4>` is the title alone, a paid ad's logo
has the company as its `alt`, the `jix_robotjob--area` span is the town and
the paragraphs are an excerpt of the ad, handed over as `teaser`. A crawled ad
("robot job") has no logo, so its company is what follows the title's last
comma, or '' when there is none. An answer with no `<channel>` is not a search
result and raises, since an empty list would read as a search that ran out.
"""
from __future__ import annotations
import html
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable
from typing import Any
from urllib.parse import urlencode
from desk.sources import strip_html
from ._words import remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._contract import Ask, Fields, Filters, Place, Reads
NAME = "jobindex"
LABEL = "Jobindex"
RANK = 10
READS = Reads(
scope="search", listing="paged", page_size=20, ceiling=200,
text="excerpt", complete="never",
filters=Filters(query="one", place="one"),
places="one",
fields=Fields(posted="trusted", country="stated"),
robots="read though robots.txt disallows sort=, page= and geoareaid=: owner, 2026-09-30")
#: Saved param -> the filter dimension it fills.
PARAMS = {"q": "query", "area": "place"}
RSS = "https://www.jobindex.dk/jobsoegning.rss"
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
#: against the titles its feeds answer with. 0 is all of Denmark.
AREAS: dict[int, str] = {
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
8: "Aarhus", 7: "Aalborg",
}
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
def allowed(slug: str) -> Callable[[str], bool]:
return only("www.jobindex.dk")
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
"""The saved params of a search: `q`, and `area` when it is a number
other than 0."""
out: dict[str, Any] = {"q": params["q"]}
if params.get("area"):
out["area"] = int(params["area"])
return out
def _text(fragment: str) -> str:
"""HTML to plain text. Jobindex breaks titles after a slash with a
zero-width space, which would otherwise end up in `title_key`."""
return html.unescape(strip_html(fragment)).replace("​", "").strip()
def _query(ask: Ask, number: int) -> str:
query: dict[str, Any] = {"q": ask.query or str(ask.params.get("q") or ""),
"sort": "date"}
area = ask.places[0] if ask.places else ask.params.get("area")
if area:
query["geoareaid"] = area
return RSS + "?" + urlencode(query | {"page": number})
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search, newest first; the cursor is the page number.
The next page follows a full page (counted before any item is dropped, so
a dropped item does not pass for the end) while the 200-result ceiling is
not reached."""
number = int(cursor or 1)
root = ET.fromstring(get(_query(ask, number)))
channel = root.find("channel")
if channel is None:
raise ValueError("jobindex answered without an RSS channel")
match = _SEARCH_AREA.search(channel.findtext("title") or "")
items = [{
"id": (item.findtext("link") or "").strip().rstrip("/").rsplit("/", 1)[-1],
"link": (item.findtext("link") or "").strip(),
"title": (item.findtext("title") or "").strip(),
"description": item.findtext("description") or "",
"published": item.findtext("pubDate"),
} for item in channel.findall("item")]
decoded = {"area": match.group(1).strip() if match else "", "items": items}
size, ceiling = READS.page_size or 20, READS.ceiling or 200
more = len(items) >= size and number * size < ceiling
return decoded, [{"id": item["id"]} for item in items], None, number + 1 if more else None
def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = []
for item in decoded["items"]:
link, full_title = item["link"], item["title"]
if not full_title or not is_web_url(link):
continue
description = item["description"]
h4 = _H4.search(description)
title = _text(h4.group(1)) if h4 else full_title
alt = _ALT.search(description)
company = html.unescape(alt.group(1)).strip() if alt else ""
if not company and full_title.startswith(title + ", "):
company = full_title[len(title) + 2:].strip()
area = _AREA.search(description)
town = _text(area.group(1)) if area else decoded["area"]
remote = remote_word(f"{town} {title}")
places: list[Place] = [{"text": town, "remote": True if remote == "remote" else None}] \
if town else []
out.append({
"source_id": item["id"],
"title": title or full_title,
"company": company,
"url": link,
"places": places,
"location": town,
"country": "DK",
"remote": remote,
"posted": parse_dt(item["published"]),
"teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
})
return out
+130
View File
@@ -0,0 +1,130 @@
"""Platsbanken: Sweden's public employment service, through the JobTech search
API at jobsearch.api.jobtechdev.se; no key.
Checked live on 2026-09-29 and 2026-09-30, moved here from `plugins/search.py`.
`region` is the id of a län in the JobTech taxonomy (Skåne län is
`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page, and an `offset`
above 2000 is refused, so a search reads 2,100 results at most. It states
`total`. A hit has the whole ad's text, so the list has the full text. An ad
may name several workplaces: `workplace_addresses` holds one entry each, and
every one is a place of the role. `workplace_model` is mostly empty, so
`remote` comes from the headline and the model together.
"""
from __future__ import annotations
import html
import json
from collections.abc import Callable
from typing import Any
from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._words import remote_word
NAME = "platsbanken"
LABEL = "Platsbanken"
RANK = 30
READS = Reads(
scope="search", listing="paged", page_size=100, ceiling=2000,
text="listing", complete="stated",
filters=Filters(query="one", place="one"),
places="many",
fields=Fields(posted="trusted", country="stated"),
robots="read: the host publishes no robots.txt, /robots.txt answers 404 "
"(checked 2026-09-30)")
#: Saved param -> the filter dimension it fills.
PARAMS = {"q": "query", "region": "place"}
SEARCH = "https://jobsearch.api.jobtechdev.se/search"
#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region`
#: concepts (which also list regions outside Sweden). "" is all of Sweden and
#: is not a key here.
REGIONS: dict[str, str] = {
"DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län",
"K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län",
"wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län",
"MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län",
"tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län",
"CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län",
"s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län",
"EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län",
"NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län",
"zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län",
"oLT3_Q9p_3nn": "Östergötlands län",
}
def allowed(slug: str) -> Callable[[str], bool]:
return only("jobsearch.api.jobtechdev.se")
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
"""The saved params of a search: `q` and `region`, the id of a län or
empty for all of Sweden."""
region = str(params.get("region") or "").strip()
if region and region not in REGIONS:
raise ValueError("region is empty or the id of a Swedish län")
return {"q": params["q"], "region": region}
def _text(fragment: str) -> str:
return html.unescape(strip_html(fragment)).strip()
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search, newest first; the cursor is the offset. The next
page follows a full page while the offset is within the ceiling."""
region = str(ask.places[0] if ask.places else ask.params.get("region") or "")
if region and region not in REGIONS:
raise ValueError(f"platsbanken region {region!r} is not one of the ids in REGIONS")
size, ceiling = READS.page_size or 100, READS.ceiling or 2000
offset = int(cursor or 0)
query: dict[str, Any] = {"q": ask.query or str(ask.params.get("q") or ""),
"sort": "pubdate-desc", "limit": size, "offset": offset}
if region:
query["region"] = region
data = json.loads(get(SEARCH + "?" + urlencode(query)))
hits = [hit for hit in data.get("hits") or [] if isinstance(hit, dict)]
total = (data.get("total") or {}).get("value")
more = len(hits) >= size and offset + size <= ceiling
return (data, [{"id": str(hit.get("id") or "")} for hit in hits],
total if isinstance(total, int) else None, offset + size if more else None)
def _place(address: dict[str, Any]) -> Place | None:
text = (address.get("municipality") or address.get("region")
or address.get("country") or "").strip()
return {"text": text, "remote": None} if text else None
def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = []
for hit in decoded.get("hits") or []:
title = _text(hit.get("headline") or "")
url = (hit.get("webpage_url") or "").strip()
if not title or not is_web_url(url):
continue
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
places = [place for address in addresses if isinstance(address, dict)
if (place := _place(address))]
model = (hit.get("workplace_model") or {}).get("label") or ""
out.append({
"source_id": str(hit.get("id") or ""),
"title": title,
"company": ((hit.get("employer") or {}).get("name") or "").strip(),
"url": url,
"places": places,
"location": places[0]["text"] if places else "",
"country": "SE",
"remote": remote_word(f"{title} {model}"),
"posted": parse_dt(hit.get("publication_date")),
"description": _text((hit.get("description") or {}).get("text") or ""),
})
return out