Merge branch 'wp1-platsbanken'

This commit is contained in:
Wolf committed 2026-09-29 16:11:11 +02:00
commit 8aa4cfdcba
4 files changed
+257 -12

No files matched your search

+10 -3
View File
@@ -25,7 +25,7 @@ if TYPE_CHECKING:
from app.jobs.discover import Discovery
#: The providers `plugins.search` runs, and which of them needs a key.
PROVIDERS = ("jobindex", "adzuna")
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
@@ -213,7 +213,7 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", pages: int = 0) -> dict[str, Any]:
where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its
provider reads."""
if provider not in PROVIDERS:
@@ -227,13 +227,20 @@ def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
params["country"] = country.strip().lower()
if where.strip():
params["where"] = where.strip()
if provider == "platsbanken":
from plugins.search import PLATSBANKEN_REGIONS
region = region.strip()
if region and region not in PLATSBANKEN_REGIONS:
raise ValueError("region is empty or the id of a Swedish län")
params["region"] = region
if pages:
params["pages"] = int(pages)
return params
def has_key(config: Config, provider: str) -> bool:
"""Whether `provider` can run: jobindex always, adzuna with a key in the
"""Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the
config or the environment."""
if provider != "adzuna":
return True
+106 -8
View File
@@ -5,7 +5,7 @@ roles at companies nobody has listed yet, by asking a cross-company board a
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
A subscription is one saved search; its `params` are the question.
Two boards, both checked live on 2026-09-23:
Three boards, checked live on 2026-09-23 and, for Platsbanken, 2026-09-29:
jobindex Denmark's largest job board. Every search result page has an RSS
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
@@ -18,6 +18,13 @@ Two boards, both checked live on 2026-09-23:
free key from developer.adzuna.com, as `sources.search.adzuna:
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
against Adzuna's documented response and not yet run live.
platsbanken Sweden's public employment service, through the JobTech search
API at jobsearch.api.jobtechdev.se, no key needed. `region` is
the id of a län in the JobTech taxonomy (Skåne län is
`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page,
and an `offset` above 2000 is refused. It states `total`. It has
no remote flag worth reading (`workplace_model` is mostly
empty), so `remote` comes from the headline.
Considered and left out: Jobnet (its search endpoint answers 401 outside the
browser session of its own site) and Jooble (a key needs a manual
@@ -26,10 +33,10 @@ application, and its API host answered a Cloudflare challenge).
A search result list is never a board's complete list, so a role missing from
one run is not closed here. A search that read everything that matches was
complete, and the pull closes what it stopped listing; the rest close once no
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states
how many results match, so an Adzuna search is complete when it read that
many; Jobindex states no count, so a Jobindex search is complete when a page
within the page limit came back short.
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna and
Platsbanken state how many results match, so their searches are complete when
they read that many; Jobindex states no count, so a Jobindex search is
complete when a page within the page limit came back short.
"""
from __future__ import annotations
@@ -62,7 +69,7 @@ TIMEOUT = 15.0
DEFAULT_PAGES = 5
MAX_PAGES = 10
PROVIDERS = ("jobindex", "adzuna")
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
#: What every provider turns a result into.
Row = dict[str, Any]
@@ -111,7 +118,7 @@ _SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
#: What a Danish ad's town line says about where the work is done:
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def _remote(text: str) -> str | None:
@@ -292,10 +299,101 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any],
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
PLATSBANKEN_SEARCH = "https://jobsearch.api.jobtechdev.se/search"
PLATSBANKEN_PAGE = 100
#: The API refuses an `offset` above 2000, which no page limit reaches.
PLATSBANKEN_MAX_OFFSET = 2000
#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region`
#: concepts (which also list regions outside Sweden). "" is all of Sweden and
#: is not a key here.
PLATSBANKEN_REGIONS: dict[str, str] = {
"DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län",
"K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län",
"wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län",
"MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län",
"tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län",
"CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län",
"s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län",
"EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län",
"NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län",
"zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län",
"oLT3_Q9p_3nn": "Östergötlands län",
}
def parse_platsbanken(body: bytes) -> list[Row]:
"""One page of JobTech's `/search`: `{"total": {"value"}, "hits": [...]}`.
A hit's `webpage_url` is Platsbanken's own page for the ad. `remote` is
read from the headline and from `workplace_model`, which the ads mostly
leave empty."""
return _platsbanken_rows(json.loads(body))
def _platsbanken_rows(data: dict[str, Any]) -> list[Row]:
rows = []
for hit in data.get("hits") or []:
title = _text(hit.get("headline") or "")
url = (hit.get("webpage_url") or "").strip()
if not title or not _web_url(url):
continue
address = hit.get("workplace_address") or {}
model = (hit.get("workplace_model") or {}).get("label") or ""
rows.append({
"id": str(hit.get("id") or ""),
"title": title,
"company": ((hit.get("employer") or {}).get("name") or "").strip(),
"place": (address.get("municipality") or address.get("region") or "").strip(),
"country": "SE",
"remote": _remote(f"{title} {model}"),
"url": url,
"posted": _parse_iso(hit.get("publication_date")),
"description": _text((hit.get("description") or {}).get("text") or ""),
"salary_min": None, "salary_max": None, "currency": "",
})
return rows
def search_platsbanken(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, newest first, until a short page, the stated
total or the page limit. Returns the rows and whether they are all the
`total` says match, as for Adzuna. `region` is a key of
`PLATSBANKEN_REGIONS`, or empty for all of Sweden."""
region = str(params.get("region") or "")
if region and region not in PLATSBANKEN_REGIONS:
raise ValueError(f"platsbanken region {region!r} is not one of the ids in "
"PLATSBANKEN_REGIONS")
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "pubdate-desc",
"limit": PLATSBANKEN_PAGE}
if region:
query["region"] = region
rows: list[Row] = []
seen: set[str] = set()
total: Any = None
ran_out = False
for page in range(_pages(params)):
offset = page * PLATSBANKEN_PAGE
if offset > PLATSBANKEN_MAX_OFFSET:
break
data = json.loads(_get(client, PLATSBANKEN_SEARCH, query | {"offset": offset}))
hits = data.get("hits") or []
total = (data.get("total") or {}).get("value", total)
seen |= {str(hit.get("id") or "") for hit in hits}
rows += _platsbanken_rows(data)
if len(hits) < PLATSBANKEN_PAGE:
ran_out = True
break
if isinstance(total, int) and len(seen) >= total:
break
return rows, (len(seen) == total) if isinstance(total, int) else ran_out
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
tuple[list[Row], bool]]] = {
"jobindex": search_jobindex,
"adzuna": search_adzuna,
"platsbanken": search_platsbanken,
}
@@ -321,7 +419,7 @@ def run_search(target: str, params: dict[str, Any],
class SearchSource(Source):
name = "search"
description = "Saved searches on cross-company job boards: Jobindex, Adzuna."
description = "Saved searches on cross-company job boards: Jobindex, Adzuna, Platsbanken."
#: Entries are `{"url": target, "label": name, "params": {...}}`.
subscription_field = "searches"
+125 -1
View File
@@ -1,6 +1,7 @@
"""The `search` plugin, `country_of`, and what the pull and the CLI do with
searches. No network: Jobindex's feed is a trimmed copy of a real one
(2026-09-23), Adzuna's is written from its documented response."""
(2026-09-23), Adzuna's is written from its documented response, Platsbanken's
is trimmed from a live answer (2026-09-29)."""
from __future__ import annotations
@@ -230,9 +231,132 @@ def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) ->
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
#: Two hits of a live `/search?q=data science&region=CaRE_1nn_cSU` (2026-09-29),
#: trimmed to the fields the parser reads, and a third the way an ad with no
#: municipality and a remote headline comes.
PLATSBANKEN = {
"total": {"value": 32},
"hits": [
{"id": "31457942", "headline": "Postdoctoral Researcher",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942",
"publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59",
"description": {"text": "The European Spallation Source (ESS) is one of the largest.",
"text_formatted": "<p>The European Spallation Source (ESS)</p>"},
"employer": {"name": "European Spallation Source Eric"},
"application_details": {"url": "https://europeanspallationsource.se/careers"},
"workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"},
"workplace_address": {"municipality": "Lund", "region": "Skåne län",
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
{"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504",
"publication_date": "2026-09-03T15:46:06",
"description": {"text": "Robotics at ESS."},
"employer": {"name": "European Spallation Source Eric"},
"workplace_model": {"concept_id": None, "label": None},
"workplace_address": {"municipality": None, "region": "Skåne län",
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
{"id": "31400001", "headline": "Data scientist, distansarbete möjligt",
"webpage_url": "javascript:alert(1)", "workplace_address": {}},
{"id": "31400002", "headline": "Data scientist, distansarbete möjligt",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002",
"employer": {"name": None}, "workplace_address": {}},
],
}
def test_platsbanken_reads_title_company_place_and_date() -> None:
rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode())
assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
first, no_town, remote = rows
assert first["title"] == "Postdoctoral Researcher"
assert first["company"] == "European Spallation Source Eric"
assert first["place"] == "Lund"
assert first["country"] == "SE"
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09"
assert first["description"].startswith("The European Spallation Source")
assert first["remote"] is None
assert no_town["place"] == "Skåne län" # the region when there is no municipality
assert remote["remote"] == "remote"
assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None)
def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None,
stock: int = 250) -> list[dict[str, Any]]:
"""Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request,
stating `total`. Returns the params of each request."""
asked: list[dict[str, Any]] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
asked.append(params)
start = params["offset"]
hits = [{"id": str(n), "headline": f"Role {n}",
"webpage_url": f"https://x.example/{n}"}
for n in range(start, min(start + params["limit"], stock))]
body: dict[str, Any] = {"hits": hits}
if total is not None:
body["total"] = {"value": total}
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
return asked
@pytest.mark.parametrize(("total", "pages", "found", "complete"), [
(250, 5, 250, True), # a short third page ends it
(300, 5, 250, False), # the stated total is more than there was
(250, 2, 200, False), # stopped by the page limit
(None, 5, 250, True), # no total: a short page is the end
])
def test_a_platsbanken_search_is_complete_when_it_read_its_total(
monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int,
complete: bool) -> None:
_platsbanken(monkeypatch, total)
rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (found, complete)
def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 200, stock=200)
rows, done = search.search_platsbanken(
None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (200, True)
assert [p["offset"] for p in asked] == [0, 100]
assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU"
assert asked[0]["limit"] == search.PLATSBANKEN_PAGE
def test_a_platsbanken_search_without_a_region_sends_none(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 5, stock=5)
search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type]
assert "region" not in asked[0]
def test_a_platsbanken_search_never_asks_past_the_offset_limit(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 5000, stock=5000)
monkeypatch.setattr(search, "MAX_PAGES", 30)
_rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type]
assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET
assert not done
def test_a_platsbanken_search_refuses_an_unknown_region() -> None:
with pytest.raises(ValueError, match="region"):
search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type]
def test_platsbanken_regions_are_the_21_lan() -> None:
assert len(search.PLATSBANKEN_REGIONS) == 21
assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län"
def test_provider_of() -> None:
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
assert search.provider_of("adzuna:berlin") == "adzuna"
assert search.provider_of("platsbanken:skane-data") == "platsbanken"
assert search.provider_of("indeed:x") == ""
assert search.provider_of("jobindex:") == ""
+16
View File
@@ -303,3 +303,19 @@ def test_replacing_or_detaching_a_board_forgets_that_it_was_failing(
store.state[FAILING + "ashby:acme"] = '{"since": "2026-09-01T00:00:00+00:00", "reason": "timeout"}'
sources.detach_board(store, "acme")
assert failing_boards(store) == {}
def test_search_params_keeps_only_what_the_provider_reads() -> None:
assert sources.search_params("jobindex", " python ", area=1, region="CaRE_1nn_cSU") == {
"q": "python", "area": 1}
assert sources.search_params("platsbanken", "data", area=1, country="de") == {
"q": "data", "region": ""}
assert sources.search_params("platsbanken", "data", region=" CaRE_1nn_cSU ", pages=3) == {
"q": "data", "region": "CaRE_1nn_cSU", "pages": 3}
def test_search_params_refuses_a_region_that_is_no_swedish_lan() -> None:
with pytest.raises(ValueError, match="region"):
sources.search_params("platsbanken", "data", region="zYpL_aGk_QY8") # Midland
with pytest.raises(ValueError, match="provider"):
sources.search_params("indeed", "data")