From 00e111961e40bae6d37c5cac979c50483e203ca2 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 16:10:45 +0200 Subject: [PATCH] Add Platsbanken as a third search provider Co-Authored-By: Claude Sonnet 5.5 --- app/jobs/sources.py | 13 ++++- plugins/search.py | 114 +++++++++++++++++++++++++++++++++++--- tests/test_search.py | 126 +++++++++++++++++++++++++++++++++++++++++- tests/test_sources.py | 16 ++++++ 4 files changed, 257 insertions(+), 12 deletions(-) diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 26b6daf..4eedbea 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from app.jobs.discover import Discovery #: The providers `plugins.search` runs, and which of them needs a key. -PROVIDERS = ("jobindex", "adzuna") +PROVIDERS = ("jobindex", "adzuna", "platsbanken") def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]: @@ -213,7 +213,7 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: def search_params(provider: str, q: str, *, area: int = 0, country: str = "", - where: str = "", pages: int = 0) -> dict[str, Any]: + where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]: """The `params` a search subscription stores, with only the keys its provider reads.""" if provider not in PROVIDERS: @@ -227,13 +227,20 @@ def search_params(provider: str, q: str, *, area: int = 0, country: str = "", params["country"] = country.strip().lower() if where.strip(): params["where"] = where.strip() + if provider == "platsbanken": + from plugins.search import PLATSBANKEN_REGIONS + + region = region.strip() + if region and region not in PLATSBANKEN_REGIONS: + raise ValueError("region is empty or the id of a Swedish län") + params["region"] = region if pages: params["pages"] = int(pages) return params def has_key(config: Config, provider: str) -> bool: - """Whether `provider` can run: jobindex always, adzuna with a key in the + """Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the config or the environment.""" if provider != "adzuna": return True diff --git a/plugins/search.py b/plugins/search.py index 0c6b2f3..55de4a4 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -5,7 +5,7 @@ roles at companies nobody has listed yet, by asking a cross-company board a saved question: "backend developer" in Region Hovedstaden, "python" in Berlin. A subscription is one saved search; its `params` are the question. -Two boards, both checked live on 2026-09-23: +Three boards, checked live on 2026-09-23 and, for Platsbanken, 2026-09-29: jobindex Denmark's largest job board. Every search result page has an RSS twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's @@ -18,6 +18,13 @@ Two boards, both checked live on 2026-09-23: free key from developer.adzuna.com, as `sources.search.adzuna: {app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built against Adzuna's documented response and not yet run live. + platsbanken Sweden's public employment service, through the JobTech search + API at jobsearch.api.jobtechdev.se, no key needed. `region` is + the id of a län in the JobTech taxonomy (Skåne län is + `CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page, + and an `offset` above 2000 is refused. It states `total`. It has + no remote flag worth reading (`workplace_model` is mostly + empty), so `remote` comes from the headline. Considered and left out: Jobnet (its search endpoint answers 401 outside the browser session of its own site) and Jooble (a key needs a manual @@ -26,10 +33,10 @@ application, and its API host answered a Cloudflare challenge). A search result list is never a board's complete list, so a role missing from one run is not closed here. A search that read everything that matches was complete, and the pull closes what it stopped listing; the rest close once no -search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states -how many results match, so an Adzuna search is complete when it read that -many; Jobindex states no count, so a Jobindex search is complete when a page -within the page limit came back short. +search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna and +Platsbanken state how many results match, so their searches are complete when +they read that many; Jobindex states no count, so a Jobindex search is +complete when a page within the page limit came back short. """ from __future__ import annotations @@ -62,7 +69,7 @@ TIMEOUT = 15.0 DEFAULT_PAGES = 5 MAX_PAGES = 10 -PROVIDERS = ("jobindex", "adzuna") +PROVIDERS = ("jobindex", "adzuna", "platsbanken") #: What every provider turns a result into. Row = dict[str, Any] @@ -111,7 +118,7 @@ _SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende") #: What a Danish ad's town line says about where the work is done: #: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". _HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) -_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE) +_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE) def _remote(text: str) -> str | None: @@ -292,10 +299,101 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any], return rows, (len(seen) == count) if isinstance(count, int) else ran_out +PLATSBANKEN_SEARCH = "https://jobsearch.api.jobtechdev.se/search" +PLATSBANKEN_PAGE = 100 +#: The API refuses an `offset` above 2000, which no page limit reaches. +PLATSBANKEN_MAX_OFFSET = 2000 + +#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region` +#: concepts (which also list regions outside Sweden). "" is all of Sweden and +#: is not a key here. +PLATSBANKEN_REGIONS: dict[str, str] = { + "DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län", + "K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län", + "wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län", + "MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län", + "tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län", + "CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län", + "s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län", + "EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län", + "NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län", + "zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län", + "oLT3_Q9p_3nn": "Östergötlands län", +} + + +def parse_platsbanken(body: bytes) -> list[Row]: + """One page of JobTech's `/search`: `{"total": {"value"}, "hits": [...]}`. + A hit's `webpage_url` is Platsbanken's own page for the ad. `remote` is + read from the headline and from `workplace_model`, which the ads mostly + leave empty.""" + return _platsbanken_rows(json.loads(body)) + + +def _platsbanken_rows(data: dict[str, Any]) -> list[Row]: + rows = [] + for hit in data.get("hits") or []: + title = _text(hit.get("headline") or "") + url = (hit.get("webpage_url") or "").strip() + if not title or not _web_url(url): + continue + address = hit.get("workplace_address") or {} + model = (hit.get("workplace_model") or {}).get("label") or "" + rows.append({ + "id": str(hit.get("id") or ""), + "title": title, + "company": ((hit.get("employer") or {}).get("name") or "").strip(), + "place": (address.get("municipality") or address.get("region") or "").strip(), + "country": "SE", + "remote": _remote(f"{title} {model}"), + "url": url, + "posted": _parse_iso(hit.get("publication_date")), + "description": _text((hit.get("description") or {}).get("text") or ""), + "salary_min": None, "salary_max": None, "currency": "", + }) + return rows + + +def search_platsbanken(client: httpx.Client, params: dict[str, Any], + keys: dict[str, Any]) -> tuple[list[Row], bool]: + """Every page of one search, newest first, until a short page, the stated + total or the page limit. Returns the rows and whether they are all the + `total` says match, as for Adzuna. `region` is a key of + `PLATSBANKEN_REGIONS`, or empty for all of Sweden.""" + region = str(params.get("region") or "") + if region and region not in PLATSBANKEN_REGIONS: + raise ValueError(f"platsbanken region {region!r} is not one of the ids in " + "PLATSBANKEN_REGIONS") + query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "pubdate-desc", + "limit": PLATSBANKEN_PAGE} + if region: + query["region"] = region + rows: list[Row] = [] + seen: set[str] = set() + total: Any = None + ran_out = False + for page in range(_pages(params)): + offset = page * PLATSBANKEN_PAGE + if offset > PLATSBANKEN_MAX_OFFSET: + break + data = json.loads(_get(client, PLATSBANKEN_SEARCH, query | {"offset": offset})) + hits = data.get("hits") or [] + total = (data.get("total") or {}).get("value", total) + seen |= {str(hit.get("id") or "") for hit in hits} + rows += _platsbanken_rows(data) + if len(hits) < PLATSBANKEN_PAGE: + ran_out = True + break + if isinstance(total, int) and len(seen) >= total: + break + return rows, (len(seen) == total) if isinstance(total, int) else ran_out + + _SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]], tuple[list[Row], bool]]] = { "jobindex": search_jobindex, "adzuna": search_adzuna, + "platsbanken": search_platsbanken, } @@ -321,7 +419,7 @@ def run_search(target: str, params: dict[str, Any], class SearchSource(Source): name = "search" - description = "Saved searches on cross-company job boards: Jobindex, Adzuna." + description = "Saved searches on cross-company job boards: Jobindex, Adzuna, Platsbanken." #: Entries are `{"url": target, "label": name, "params": {...}}`. subscription_field = "searches" diff --git a/tests/test_search.py b/tests/test_search.py index 813bff4..15fd7a3 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1,6 +1,7 @@ """The `search` plugin, `country_of`, and what the pull and the CLI do with searches. No network: Jobindex's feed is a trimmed copy of a real one -(2026-09-23), Adzuna's is written from its documented response.""" +(2026-09-23), Adzuna's is written from its documented response, Platsbanken's +is trimmed from a live answer (2026-09-29).""" from __future__ import annotations @@ -230,9 +231,132 @@ def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True) +#: Two hits of a live `/search?q=data science®ion=CaRE_1nn_cSU` (2026-09-29), +#: trimmed to the fields the parser reads, and a third the way an ad with no +#: municipality and a remote headline comes. +PLATSBANKEN = { + "total": {"value": 32}, + "hits": [ + {"id": "31457942", "headline": "Postdoctoral Researcher", + "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942", + "publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59", + "description": {"text": "The European Spallation Source (ESS) is one of the largest.", + "text_formatted": "

The European Spallation Source (ESS)

"}, + "employer": {"name": "European Spallation Source Eric"}, + "application_details": {"url": "https://europeanspallationsource.se/careers"}, + "workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"}, + "workplace_address": {"municipality": "Lund", "region": "Skåne län", + "region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}}, + {"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision", + "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504", + "publication_date": "2026-09-03T15:46:06", + "description": {"text": "Robotics at ESS."}, + "employer": {"name": "European Spallation Source Eric"}, + "workplace_model": {"concept_id": None, "label": None}, + "workplace_address": {"municipality": None, "region": "Skåne län", + "region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}}, + {"id": "31400001", "headline": "Data scientist, distansarbete möjligt", + "webpage_url": "javascript:alert(1)", "workplace_address": {}}, + {"id": "31400002", "headline": "Data scientist, distansarbete möjligt", + "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002", + "employer": {"name": None}, "workplace_address": {}}, + ], +} + + +def test_platsbanken_reads_title_company_place_and_date() -> None: + rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode()) + assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused + first, no_town, remote = rows + assert first["title"] == "Postdoctoral Researcher" + assert first["company"] == "European Spallation Source Eric" + assert first["place"] == "Lund" + assert first["country"] == "SE" + assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942" + assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09" + assert first["description"].startswith("The European Spallation Source") + assert first["remote"] is None + assert no_town["place"] == "Skåne län" # the region when there is no municipality + assert remote["remote"] == "remote" + assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None) + + +def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None, + stock: int = 250) -> list[dict[str, Any]]: + """Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request, + stating `total`. Returns the params of each request.""" + asked: list[dict[str, Any]] = [] + + def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: + asked.append(params) + start = params["offset"] + hits = [{"id": str(n), "headline": f"Role {n}", + "webpage_url": f"https://x.example/{n}"} + for n in range(start, min(start + params["limit"], stock))] + body: dict[str, Any] = {"hits": hits} + if total is not None: + body["total"] = {"value": total} + return json.dumps(body).encode() + + monkeypatch.setattr(search, "_get", fake_get) + return asked + + +@pytest.mark.parametrize(("total", "pages", "found", "complete"), [ + (250, 5, 250, True), # a short third page ends it + (300, 5, 250, False), # the stated total is more than there was + (250, 2, 200, False), # stopped by the page limit + (None, 5, 250, True), # no total: a short page is the end +]) +def test_a_platsbanken_search_is_complete_when_it_read_its_total( + monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int, + complete: bool) -> None: + _platsbanken(monkeypatch, total) + rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type] + assert (len(rows), done) == (found, complete) + + +def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset( + monkeypatch: pytest.MonkeyPatch) -> None: + asked = _platsbanken(monkeypatch, 200, stock=200) + rows, done = search.search_platsbanken( + None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type] + assert (len(rows), done) == (200, True) + assert [p["offset"] for p in asked] == [0, 100] + assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU" + assert asked[0]["limit"] == search.PLATSBANKEN_PAGE + + +def test_a_platsbanken_search_without_a_region_sends_none( + monkeypatch: pytest.MonkeyPatch) -> None: + asked = _platsbanken(monkeypatch, 5, stock=5) + search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type] + assert "region" not in asked[0] + + +def test_a_platsbanken_search_never_asks_past_the_offset_limit( + monkeypatch: pytest.MonkeyPatch) -> None: + asked = _platsbanken(monkeypatch, 5000, stock=5000) + monkeypatch.setattr(search, "MAX_PAGES", 30) + _rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type] + assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET + assert not done + + +def test_a_platsbanken_search_refuses_an_unknown_region() -> None: + with pytest.raises(ValueError, match="region"): + search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type] + + +def test_platsbanken_regions_are_the_21_lan() -> None: + assert len(search.PLATSBANKEN_REGIONS) == 21 + assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län" + + def test_provider_of() -> None: assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("adzuna:berlin") == "adzuna" + assert search.provider_of("platsbanken:skane-data") == "platsbanken" assert search.provider_of("indeed:x") == "" assert search.provider_of("jobindex:") == "" diff --git a/tests/test_sources.py b/tests/test_sources.py index edea9e1..515e326 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -303,3 +303,19 @@ def test_replacing_or_detaching_a_board_forgets_that_it_was_failing( store.state[FAILING + "ashby:acme"] = '{"since": "2026-09-01T00:00:00+00:00", "reason": "timeout"}' sources.detach_board(store, "acme") assert failing_boards(store) == {} + + +def test_search_params_keeps_only_what_the_provider_reads() -> None: + assert sources.search_params("jobindex", " python ", area=1, region="CaRE_1nn_cSU") == { + "q": "python", "area": 1} + assert sources.search_params("platsbanken", "data", area=1, country="de") == { + "q": "data", "region": ""} + assert sources.search_params("platsbanken", "data", region=" CaRE_1nn_cSU ", pages=3) == { + "q": "data", "region": "CaRE_1nn_cSU", "pages": 3} + + +def test_search_params_refuses_a_region_that_is_no_swedish_lan() -> None: + with pytest.raises(ValueError, match="region"): + sources.search_params("platsbanken", "data", region="zYpL_aGk_QY8") # Midland + with pytest.raises(ValueError, match="provider"): + sources.search_params("indeed", "data")