Merge branch 'wp1-platsbanken'

This commit is contained in:
Wolf committed 2026-09-29 16:11:11 +02:00
commit 8aa4cfdcba
4 files changed
+257 -12

No files matched your search

+10 -3
View File
@@ -25,7 +25,7 @@ if TYPE_CHECKING:
from app.jobs.discover import Discovery from app.jobs.discover import Discovery
#: The providers `plugins.search` runs, and which of them needs a key. #: The providers `plugins.search` runs, and which of them needs a key.
PROVIDERS = ("jobindex", "adzuna") PROVIDERS = ("jobindex", "adzuna", "platsbanken")
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]: def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
@@ -213,7 +213,7 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
def search_params(provider: str, q: str, *, area: int = 0, country: str = "", def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", pages: int = 0) -> dict[str, Any]: where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its """The `params` a search subscription stores, with only the keys its
provider reads.""" provider reads."""
if provider not in PROVIDERS: if provider not in PROVIDERS:
@@ -227,13 +227,20 @@ def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
params["country"] = country.strip().lower() params["country"] = country.strip().lower()
if where.strip(): if where.strip():
params["where"] = where.strip() params["where"] = where.strip()
if provider == "platsbanken":
from plugins.search import PLATSBANKEN_REGIONS
region = region.strip()
if region and region not in PLATSBANKEN_REGIONS:
raise ValueError("region is empty or the id of a Swedish län")
params["region"] = region
if pages: if pages:
params["pages"] = int(pages) params["pages"] = int(pages)
return params return params
def has_key(config: Config, provider: str) -> bool: def has_key(config: Config, provider: str) -> bool:
"""Whether `provider` can run: jobindex always, adzuna with a key in the """Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the
config or the environment.""" config or the environment."""
if provider != "adzuna": if provider != "adzuna":
return True return True
+106 -8
View File
@@ -5,7 +5,7 @@ roles at companies nobody has listed yet, by asking a cross-company board a
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin. saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
A subscription is one saved search; its `params` are the question. A subscription is one saved search; its `params` are the question.
Two boards, both checked live on 2026-09-23: Three boards, checked live on 2026-09-23 and, for Platsbanken, 2026-09-29:
jobindex Denmark's largest job board. Every search result page has an RSS jobindex Denmark's largest job board. Every search result page has an RSS
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
@@ -18,6 +18,13 @@ Two boards, both checked live on 2026-09-23:
free key from developer.adzuna.com, as `sources.search.adzuna: free key from developer.adzuna.com, as `sources.search.adzuna:
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built {app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
against Adzuna's documented response and not yet run live. against Adzuna's documented response and not yet run live.
platsbanken Sweden's public employment service, through the JobTech search
API at jobsearch.api.jobtechdev.se, no key needed. `region` is
the id of a län in the JobTech taxonomy (Skåne län is
`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page,
and an `offset` above 2000 is refused. It states `total`. It has
no remote flag worth reading (`workplace_model` is mostly
empty), so `remote` comes from the headline.
Considered and left out: Jobnet (its search endpoint answers 401 outside the Considered and left out: Jobnet (its search endpoint answers 401 outside the
browser session of its own site) and Jooble (a key needs a manual browser session of its own site) and Jooble (a key needs a manual
@@ -26,10 +33,10 @@ application, and its API host answered a Cloudflare challenge).
A search result list is never a board's complete list, so a role missing from A search result list is never a board's complete list, so a role missing from
one run is not closed here. A search that read everything that matches was one run is not closed here. A search that read everything that matches was
complete, and the pull closes what it stopped listing; the rest close once no complete, and the pull closes what it stopped listing; the rest close once no
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna and
how many results match, so an Adzuna search is complete when it read that Platsbanken state how many results match, so their searches are complete when
many; Jobindex states no count, so a Jobindex search is complete when a page they read that many; Jobindex states no count, so a Jobindex search is
within the page limit came back short. complete when a page within the page limit came back short.
""" """
from __future__ import annotations from __future__ import annotations
@@ -62,7 +69,7 @@ TIMEOUT = 15.0
DEFAULT_PAGES = 5 DEFAULT_PAGES = 5
MAX_PAGES = 10 MAX_PAGES = 10
PROVIDERS = ("jobindex", "adzuna") PROVIDERS = ("jobindex", "adzuna", "platsbanken")
#: What every provider turns a result into. #: What every provider turns a result into.
Row = dict[str, Any] Row = dict[str, Any]
@@ -111,7 +118,7 @@ _SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
#: What a Danish ad's town line says about where the work is done: #: What a Danish ad's town line says about where the work is done:
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". #: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) _HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE) _REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def _remote(text: str) -> str | None: def _remote(text: str) -> str | None:
@@ -292,10 +299,101 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any],
return rows, (len(seen) == count) if isinstance(count, int) else ran_out return rows, (len(seen) == count) if isinstance(count, int) else ran_out
PLATSBANKEN_SEARCH = "https://jobsearch.api.jobtechdev.se/search"
PLATSBANKEN_PAGE = 100
#: The API refuses an `offset` above 2000, which no page limit reaches.
PLATSBANKEN_MAX_OFFSET = 2000
#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region`
#: concepts (which also list regions outside Sweden). "" is all of Sweden and
#: is not a key here.
PLATSBANKEN_REGIONS: dict[str, str] = {
"DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län",
"K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län",
"wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län",
"MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län",
"tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län",
"CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län",
"s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län",
"EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län",
"NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län",
"zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län",
"oLT3_Q9p_3nn": "Östergötlands län",
}
def parse_platsbanken(body: bytes) -> list[Row]:
"""One page of JobTech's `/search`: `{"total": {"value"}, "hits": [...]}`.
A hit's `webpage_url` is Platsbanken's own page for the ad. `remote` is
read from the headline and from `workplace_model`, which the ads mostly
leave empty."""
return _platsbanken_rows(json.loads(body))
def _platsbanken_rows(data: dict[str, Any]) -> list[Row]:
rows = []
for hit in data.get("hits") or []:
title = _text(hit.get("headline") or "")
url = (hit.get("webpage_url") or "").strip()
if not title or not _web_url(url):
continue
address = hit.get("workplace_address") or {}
model = (hit.get("workplace_model") or {}).get("label") or ""
rows.append({
"id": str(hit.get("id") or ""),
"title": title,
"company": ((hit.get("employer") or {}).get("name") or "").strip(),
"place": (address.get("municipality") or address.get("region") or "").strip(),
"country": "SE",
"remote": _remote(f"{title} {model}"),
"url": url,
"posted": _parse_iso(hit.get("publication_date")),
"description": _text((hit.get("description") or {}).get("text") or ""),
"salary_min": None, "salary_max": None, "currency": "",
})
return rows
def search_platsbanken(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, newest first, until a short page, the stated
total or the page limit. Returns the rows and whether they are all the
`total` says match, as for Adzuna. `region` is a key of
`PLATSBANKEN_REGIONS`, or empty for all of Sweden."""
region = str(params.get("region") or "")
if region and region not in PLATSBANKEN_REGIONS:
raise ValueError(f"platsbanken region {region!r} is not one of the ids in "
"PLATSBANKEN_REGIONS")
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "pubdate-desc",
"limit": PLATSBANKEN_PAGE}
if region:
query["region"] = region
rows: list[Row] = []
seen: set[str] = set()
total: Any = None
ran_out = False
for page in range(_pages(params)):
offset = page * PLATSBANKEN_PAGE
if offset > PLATSBANKEN_MAX_OFFSET:
break
data = json.loads(_get(client, PLATSBANKEN_SEARCH, query | {"offset": offset}))
hits = data.get("hits") or []
total = (data.get("total") or {}).get("value", total)
seen |= {str(hit.get("id") or "") for hit in hits}
rows += _platsbanken_rows(data)
if len(hits) < PLATSBANKEN_PAGE:
ran_out = True
break
if isinstance(total, int) and len(seen) >= total:
break
return rows, (len(seen) == total) if isinstance(total, int) else ran_out
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]], _SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
tuple[list[Row], bool]]] = { tuple[list[Row], bool]]] = {
"jobindex": search_jobindex, "jobindex": search_jobindex,
"adzuna": search_adzuna, "adzuna": search_adzuna,
"platsbanken": search_platsbanken,
} }
@@ -321,7 +419,7 @@ def run_search(target: str, params: dict[str, Any],
class SearchSource(Source): class SearchSource(Source):
name = "search" name = "search"
description = "Saved searches on cross-company job boards: Jobindex, Adzuna." description = "Saved searches on cross-company job boards: Jobindex, Adzuna, Platsbanken."
#: Entries are `{"url": target, "label": name, "params": {...}}`. #: Entries are `{"url": target, "label": name, "params": {...}}`.
subscription_field = "searches" subscription_field = "searches"
+125 -1
View File
@@ -1,6 +1,7 @@
"""The `search` plugin, `country_of`, and what the pull and the CLI do with """The `search` plugin, `country_of`, and what the pull and the CLI do with
searches. No network: Jobindex's feed is a trimmed copy of a real one searches. No network: Jobindex's feed is a trimmed copy of a real one
(2026-09-23), Adzuna's is written from its documented response.""" (2026-09-23), Adzuna's is written from its documented response, Platsbanken's
is trimmed from a live answer (2026-09-29)."""
from __future__ import annotations from __future__ import annotations
@@ -230,9 +231,132 @@ def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) ->
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True) assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
#: Two hits of a live `/search?q=data science&region=CaRE_1nn_cSU` (2026-09-29),
#: trimmed to the fields the parser reads, and a third the way an ad with no
#: municipality and a remote headline comes.
PLATSBANKEN = {
"total": {"value": 32},
"hits": [
{"id": "31457942", "headline": "Postdoctoral Researcher",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942",
"publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59",
"description": {"text": "The European Spallation Source (ESS) is one of the largest.",
"text_formatted": "<p>The European Spallation Source (ESS)</p>"},
"employer": {"name": "European Spallation Source Eric"},
"application_details": {"url": "https://europeanspallationsource.se/careers"},
"workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"},
"workplace_address": {"municipality": "Lund", "region": "Skåne län",
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
{"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504",
"publication_date": "2026-09-03T15:46:06",
"description": {"text": "Robotics at ESS."},
"employer": {"name": "European Spallation Source Eric"},
"workplace_model": {"concept_id": None, "label": None},
"workplace_address": {"municipality": None, "region": "Skåne län",
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
{"id": "31400001", "headline": "Data scientist, distansarbete möjligt",
"webpage_url": "javascript:alert(1)", "workplace_address": {}},
{"id": "31400002", "headline": "Data scientist, distansarbete möjligt",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002",
"employer": {"name": None}, "workplace_address": {}},
],
}
def test_platsbanken_reads_title_company_place_and_date() -> None:
rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode())
assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
first, no_town, remote = rows
assert first["title"] == "Postdoctoral Researcher"
assert first["company"] == "European Spallation Source Eric"
assert first["place"] == "Lund"
assert first["country"] == "SE"
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09"
assert first["description"].startswith("The European Spallation Source")
assert first["remote"] is None
assert no_town["place"] == "Skåne län" # the region when there is no municipality
assert remote["remote"] == "remote"
assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None)
def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None,
stock: int = 250) -> list[dict[str, Any]]:
"""Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request,
stating `total`. Returns the params of each request."""
asked: list[dict[str, Any]] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
asked.append(params)
start = params["offset"]
hits = [{"id": str(n), "headline": f"Role {n}",
"webpage_url": f"https://x.example/{n}"}
for n in range(start, min(start + params["limit"], stock))]
body: dict[str, Any] = {"hits": hits}
if total is not None:
body["total"] = {"value": total}
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
return asked
@pytest.mark.parametrize(("total", "pages", "found", "complete"), [
(250, 5, 250, True), # a short third page ends it
(300, 5, 250, False), # the stated total is more than there was
(250, 2, 200, False), # stopped by the page limit
(None, 5, 250, True), # no total: a short page is the end
])
def test_a_platsbanken_search_is_complete_when_it_read_its_total(
monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int,
complete: bool) -> None:
_platsbanken(monkeypatch, total)
rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (found, complete)
def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 200, stock=200)
rows, done = search.search_platsbanken(
None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (200, True)
assert [p["offset"] for p in asked] == [0, 100]
assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU"
assert asked[0]["limit"] == search.PLATSBANKEN_PAGE
def test_a_platsbanken_search_without_a_region_sends_none(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 5, stock=5)
search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type]
assert "region" not in asked[0]
def test_a_platsbanken_search_never_asks_past_the_offset_limit(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 5000, stock=5000)
monkeypatch.setattr(search, "MAX_PAGES", 30)
_rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type]
assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET
assert not done
def test_a_platsbanken_search_refuses_an_unknown_region() -> None:
with pytest.raises(ValueError, match="region"):
search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type]
def test_platsbanken_regions_are_the_21_lan() -> None:
assert len(search.PLATSBANKEN_REGIONS) == 21
assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län"
def test_provider_of() -> None: def test_provider_of() -> None:
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
assert search.provider_of("adzuna:berlin") == "adzuna" assert search.provider_of("adzuna:berlin") == "adzuna"
assert search.provider_of("platsbanken:skane-data") == "platsbanken"
assert search.provider_of("indeed:x") == "" assert search.provider_of("indeed:x") == ""
assert search.provider_of("jobindex:") == "" assert search.provider_of("jobindex:") == ""
+16
View File
@@ -303,3 +303,19 @@ def test_replacing_or_detaching_a_board_forgets_that_it_was_failing(
store.state[FAILING + "ashby:acme"] = '{"since": "2026-09-01T00:00:00+00:00", "reason": "timeout"}' store.state[FAILING + "ashby:acme"] = '{"since": "2026-09-01T00:00:00+00:00", "reason": "timeout"}'
sources.detach_board(store, "acme") sources.detach_board(store, "acme")
assert failing_boards(store) == {} assert failing_boards(store) == {}
def test_search_params_keeps_only_what_the_provider_reads() -> None:
assert sources.search_params("jobindex", " python ", area=1, region="CaRE_1nn_cSU") == {
"q": "python", "area": 1}
assert sources.search_params("platsbanken", "data", area=1, country="de") == {
"q": "data", "region": ""}
assert sources.search_params("platsbanken", "data", region=" CaRE_1nn_cSU ", pages=3) == {
"q": "data", "region": "CaRE_1nn_cSU", "pages": 3}
def test_search_params_refuses_a_region_that_is_no_swedish_lan() -> None:
with pytest.raises(ValueError, match="region"):
sources.search_params("platsbanken", "data", region="zYpL_aGk_QY8") # Midland
with pytest.raises(ValueError, match="provider"):
sources.search_params("indeed", "data")