Merge branch 'wp1-platsbanken'
This commit is contained in:
commit
8aa4cfdcba
4 files changed
+257
-12
No files matched your search
+10
-3
@@ -25,7 +25,7 @@ if TYPE_CHECKING:
|
||||
from app.jobs.discover import Discovery
|
||||
|
||||
#: The providers `plugins.search` runs, and which of them needs a key.
|
||||
PROVIDERS = ("jobindex", "adzuna")
|
||||
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
|
||||
|
||||
|
||||
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
|
||||
@@ -213,7 +213,7 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
||||
where: str = "", pages: int = 0) -> dict[str, Any]:
|
||||
where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]:
|
||||
"""The `params` a search subscription stores, with only the keys its
|
||||
provider reads."""
|
||||
if provider not in PROVIDERS:
|
||||
@@ -227,13 +227,20 @@ def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
||||
params["country"] = country.strip().lower()
|
||||
if where.strip():
|
||||
params["where"] = where.strip()
|
||||
if provider == "platsbanken":
|
||||
from plugins.search import PLATSBANKEN_REGIONS
|
||||
|
||||
region = region.strip()
|
||||
if region and region not in PLATSBANKEN_REGIONS:
|
||||
raise ValueError("region is empty or the id of a Swedish län")
|
||||
params["region"] = region
|
||||
if pages:
|
||||
params["pages"] = int(pages)
|
||||
return params
|
||||
|
||||
|
||||
def has_key(config: Config, provider: str) -> bool:
|
||||
"""Whether `provider` can run: jobindex always, adzuna with a key in the
|
||||
"""Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the
|
||||
config or the environment."""
|
||||
if provider != "adzuna":
|
||||
return True
|
||||
|
||||
+106
-8
@@ -5,7 +5,7 @@ roles at companies nobody has listed yet, by asking a cross-company board a
|
||||
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
|
||||
A subscription is one saved search; its `params` are the question.
|
||||
|
||||
Two boards, both checked live on 2026-09-23:
|
||||
Three boards, checked live on 2026-09-23 and, for Platsbanken, 2026-09-29:
|
||||
|
||||
jobindex Denmark's largest job board. Every search result page has an RSS
|
||||
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
|
||||
@@ -18,6 +18,13 @@ Two boards, both checked live on 2026-09-23:
|
||||
free key from developer.adzuna.com, as `sources.search.adzuna:
|
||||
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
|
||||
against Adzuna's documented response and not yet run live.
|
||||
platsbanken Sweden's public employment service, through the JobTech search
|
||||
API at jobsearch.api.jobtechdev.se, no key needed. `region` is
|
||||
the id of a län in the JobTech taxonomy (Skåne län is
|
||||
`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page,
|
||||
and an `offset` above 2000 is refused. It states `total`. It has
|
||||
no remote flag worth reading (`workplace_model` is mostly
|
||||
empty), so `remote` comes from the headline.
|
||||
|
||||
Considered and left out: Jobnet (its search endpoint answers 401 outside the
|
||||
browser session of its own site) and Jooble (a key needs a manual
|
||||
@@ -26,10 +33,10 @@ application, and its API host answered a Cloudflare challenge).
|
||||
A search result list is never a board's complete list, so a role missing from
|
||||
one run is not closed here. A search that read everything that matches was
|
||||
complete, and the pull closes what it stopped listing; the rest close once no
|
||||
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states
|
||||
how many results match, so an Adzuna search is complete when it read that
|
||||
many; Jobindex states no count, so a Jobindex search is complete when a page
|
||||
within the page limit came back short.
|
||||
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna and
|
||||
Platsbanken state how many results match, so their searches are complete when
|
||||
they read that many; Jobindex states no count, so a Jobindex search is
|
||||
complete when a page within the page limit came back short.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -62,7 +69,7 @@ TIMEOUT = 15.0
|
||||
DEFAULT_PAGES = 5
|
||||
MAX_PAGES = 10
|
||||
|
||||
PROVIDERS = ("jobindex", "adzuna")
|
||||
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
|
||||
|
||||
#: What every provider turns a result into.
|
||||
Row = dict[str, Any]
|
||||
@@ -111,7 +118,7 @@ _SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
||||
#: What a Danish ad's town line says about where the work is done:
|
||||
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
||||
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
|
||||
|
||||
|
||||
def _remote(text: str) -> str | None:
|
||||
@@ -292,10 +299,101 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
||||
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
|
||||
|
||||
|
||||
PLATSBANKEN_SEARCH = "https://jobsearch.api.jobtechdev.se/search"
|
||||
PLATSBANKEN_PAGE = 100
|
||||
#: The API refuses an `offset` above 2000, which no page limit reaches.
|
||||
PLATSBANKEN_MAX_OFFSET = 2000
|
||||
|
||||
#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region`
|
||||
#: concepts (which also list regions outside Sweden). "" is all of Sweden and
|
||||
#: is not a key here.
|
||||
PLATSBANKEN_REGIONS: dict[str, str] = {
|
||||
"DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län",
|
||||
"K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län",
|
||||
"wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län",
|
||||
"MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län",
|
||||
"tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län",
|
||||
"CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län",
|
||||
"s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län",
|
||||
"EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län",
|
||||
"NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län",
|
||||
"zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län",
|
||||
"oLT3_Q9p_3nn": "Östergötlands län",
|
||||
}
|
||||
|
||||
|
||||
def parse_platsbanken(body: bytes) -> list[Row]:
|
||||
"""One page of JobTech's `/search`: `{"total": {"value"}, "hits": [...]}`.
|
||||
A hit's `webpage_url` is Platsbanken's own page for the ad. `remote` is
|
||||
read from the headline and from `workplace_model`, which the ads mostly
|
||||
leave empty."""
|
||||
return _platsbanken_rows(json.loads(body))
|
||||
|
||||
|
||||
def _platsbanken_rows(data: dict[str, Any]) -> list[Row]:
|
||||
rows = []
|
||||
for hit in data.get("hits") or []:
|
||||
title = _text(hit.get("headline") or "")
|
||||
url = (hit.get("webpage_url") or "").strip()
|
||||
if not title or not _web_url(url):
|
||||
continue
|
||||
address = hit.get("workplace_address") or {}
|
||||
model = (hit.get("workplace_model") or {}).get("label") or ""
|
||||
rows.append({
|
||||
"id": str(hit.get("id") or ""),
|
||||
"title": title,
|
||||
"company": ((hit.get("employer") or {}).get("name") or "").strip(),
|
||||
"place": (address.get("municipality") or address.get("region") or "").strip(),
|
||||
"country": "SE",
|
||||
"remote": _remote(f"{title} {model}"),
|
||||
"url": url,
|
||||
"posted": _parse_iso(hit.get("publication_date")),
|
||||
"description": _text((hit.get("description") or {}).get("text") or ""),
|
||||
"salary_min": None, "salary_max": None, "currency": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def search_platsbanken(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, newest first, until a short page, the stated
|
||||
total or the page limit. Returns the rows and whether they are all the
|
||||
`total` says match, as for Adzuna. `region` is a key of
|
||||
`PLATSBANKEN_REGIONS`, or empty for all of Sweden."""
|
||||
region = str(params.get("region") or "")
|
||||
if region and region not in PLATSBANKEN_REGIONS:
|
||||
raise ValueError(f"platsbanken region {region!r} is not one of the ids in "
|
||||
"PLATSBANKEN_REGIONS")
|
||||
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "pubdate-desc",
|
||||
"limit": PLATSBANKEN_PAGE}
|
||||
if region:
|
||||
query["region"] = region
|
||||
rows: list[Row] = []
|
||||
seen: set[str] = set()
|
||||
total: Any = None
|
||||
ran_out = False
|
||||
for page in range(_pages(params)):
|
||||
offset = page * PLATSBANKEN_PAGE
|
||||
if offset > PLATSBANKEN_MAX_OFFSET:
|
||||
break
|
||||
data = json.loads(_get(client, PLATSBANKEN_SEARCH, query | {"offset": offset}))
|
||||
hits = data.get("hits") or []
|
||||
total = (data.get("total") or {}).get("value", total)
|
||||
seen |= {str(hit.get("id") or "") for hit in hits}
|
||||
rows += _platsbanken_rows(data)
|
||||
if len(hits) < PLATSBANKEN_PAGE:
|
||||
ran_out = True
|
||||
break
|
||||
if isinstance(total, int) and len(seen) >= total:
|
||||
break
|
||||
return rows, (len(seen) == total) if isinstance(total, int) else ran_out
|
||||
|
||||
|
||||
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
|
||||
tuple[list[Row], bool]]] = {
|
||||
"jobindex": search_jobindex,
|
||||
"adzuna": search_adzuna,
|
||||
"platsbanken": search_platsbanken,
|
||||
}
|
||||
|
||||
|
||||
@@ -321,7 +419,7 @@ def run_search(target: str, params: dict[str, Any],
|
||||
|
||||
class SearchSource(Source):
|
||||
name = "search"
|
||||
description = "Saved searches on cross-company job boards: Jobindex, Adzuna."
|
||||
description = "Saved searches on cross-company job boards: Jobindex, Adzuna, Platsbanken."
|
||||
#: Entries are `{"url": target, "label": name, "params": {...}}`.
|
||||
subscription_field = "searches"
|
||||
|
||||
|
||||
+125
-1
@@ -1,6 +1,7 @@
|
||||
"""The `search` plugin, `country_of`, and what the pull and the CLI do with
|
||||
searches. No network: Jobindex's feed is a trimmed copy of a real one
|
||||
(2026-09-23), Adzuna's is written from its documented response."""
|
||||
(2026-09-23), Adzuna's is written from its documented response, Platsbanken's
|
||||
is trimmed from a live answer (2026-09-29)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -230,9 +231,132 @@ def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) ->
|
||||
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
|
||||
|
||||
|
||||
#: Two hits of a live `/search?q=data science®ion=CaRE_1nn_cSU` (2026-09-29),
|
||||
#: trimmed to the fields the parser reads, and a third the way an ad with no
|
||||
#: municipality and a remote headline comes.
|
||||
PLATSBANKEN = {
|
||||
"total": {"value": 32},
|
||||
"hits": [
|
||||
{"id": "31457942", "headline": "Postdoctoral Researcher",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942",
|
||||
"publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59",
|
||||
"description": {"text": "The European Spallation Source (ESS) is one of the largest.",
|
||||
"text_formatted": "<p>The European Spallation Source (ESS)</p>"},
|
||||
"employer": {"name": "European Spallation Source Eric"},
|
||||
"application_details": {"url": "https://europeanspallationsource.se/careers"},
|
||||
"workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"},
|
||||
"workplace_address": {"municipality": "Lund", "region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
|
||||
{"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504",
|
||||
"publication_date": "2026-09-03T15:46:06",
|
||||
"description": {"text": "Robotics at ESS."},
|
||||
"employer": {"name": "European Spallation Source Eric"},
|
||||
"workplace_model": {"concept_id": None, "label": None},
|
||||
"workplace_address": {"municipality": None, "region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
|
||||
{"id": "31400001", "headline": "Data scientist, distansarbete möjligt",
|
||||
"webpage_url": "javascript:alert(1)", "workplace_address": {}},
|
||||
{"id": "31400002", "headline": "Data scientist, distansarbete möjligt",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002",
|
||||
"employer": {"name": None}, "workplace_address": {}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_platsbanken_reads_title_company_place_and_date() -> None:
|
||||
rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode())
|
||||
assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
||||
first, no_town, remote = rows
|
||||
assert first["title"] == "Postdoctoral Researcher"
|
||||
assert first["company"] == "European Spallation Source Eric"
|
||||
assert first["place"] == "Lund"
|
||||
assert first["country"] == "SE"
|
||||
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
|
||||
assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09"
|
||||
assert first["description"].startswith("The European Spallation Source")
|
||||
assert first["remote"] is None
|
||||
assert no_town["place"] == "Skåne län" # the region when there is no municipality
|
||||
assert remote["remote"] == "remote"
|
||||
assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None)
|
||||
|
||||
|
||||
def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None,
|
||||
stock: int = 250) -> list[dict[str, Any]]:
|
||||
"""Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request,
|
||||
stating `total`. Returns the params of each request."""
|
||||
asked: list[dict[str, Any]] = []
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
asked.append(params)
|
||||
start = params["offset"]
|
||||
hits = [{"id": str(n), "headline": f"Role {n}",
|
||||
"webpage_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + params["limit"], stock))]
|
||||
body: dict[str, Any] = {"hits": hits}
|
||||
if total is not None:
|
||||
body["total"] = {"value": total}
|
||||
return json.dumps(body).encode()
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
return asked
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("total", "pages", "found", "complete"), [
|
||||
(250, 5, 250, True), # a short third page ends it
|
||||
(300, 5, 250, False), # the stated total is more than there was
|
||||
(250, 2, 200, False), # stopped by the page limit
|
||||
(None, 5, 250, True), # no total: a short page is the end
|
||||
])
|
||||
def test_a_platsbanken_search_is_complete_when_it_read_its_total(
|
||||
monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int,
|
||||
complete: bool) -> None:
|
||||
_platsbanken(monkeypatch, total)
|
||||
rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type]
|
||||
assert (len(rows), done) == (found, complete)
|
||||
|
||||
|
||||
def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 200, stock=200)
|
||||
rows, done = search.search_platsbanken(
|
||||
None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type]
|
||||
assert (len(rows), done) == (200, True)
|
||||
assert [p["offset"] for p in asked] == [0, 100]
|
||||
assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU"
|
||||
assert asked[0]["limit"] == search.PLATSBANKEN_PAGE
|
||||
|
||||
|
||||
def test_a_platsbanken_search_without_a_region_sends_none(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 5, stock=5)
|
||||
search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type]
|
||||
assert "region" not in asked[0]
|
||||
|
||||
|
||||
def test_a_platsbanken_search_never_asks_past_the_offset_limit(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 5000, stock=5000)
|
||||
monkeypatch.setattr(search, "MAX_PAGES", 30)
|
||||
_rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type]
|
||||
assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET
|
||||
assert not done
|
||||
|
||||
|
||||
def test_a_platsbanken_search_refuses_an_unknown_region() -> None:
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_platsbanken_regions_are_the_21_lan() -> None:
|
||||
assert len(search.PLATSBANKEN_REGIONS) == 21
|
||||
assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län"
|
||||
|
||||
|
||||
def test_provider_of() -> None:
|
||||
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
|
||||
assert search.provider_of("adzuna:berlin") == "adzuna"
|
||||
assert search.provider_of("platsbanken:skane-data") == "platsbanken"
|
||||
assert search.provider_of("indeed:x") == ""
|
||||
assert search.provider_of("jobindex:") == ""
|
||||
|
||||
|
||||
@@ -303,3 +303,19 @@ def test_replacing_or_detaching_a_board_forgets_that_it_was_failing(
|
||||
store.state[FAILING + "ashby:acme"] = '{"since": "2026-09-01T00:00:00+00:00", "reason": "timeout"}'
|
||||
sources.detach_board(store, "acme")
|
||||
assert failing_boards(store) == {}
|
||||
|
||||
|
||||
def test_search_params_keeps_only_what_the_provider_reads() -> None:
|
||||
assert sources.search_params("jobindex", " python ", area=1, region="CaRE_1nn_cSU") == {
|
||||
"q": "python", "area": 1}
|
||||
assert sources.search_params("platsbanken", "data", area=1, country="de") == {
|
||||
"q": "data", "region": ""}
|
||||
assert sources.search_params("platsbanken", "data", region=" CaRE_1nn_cSU ", pages=3) == {
|
||||
"q": "data", "region": "CaRE_1nn_cSU", "pages": 3}
|
||||
|
||||
|
||||
def test_search_params_refuses_a_region_that_is_no_swedish_lan() -> None:
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
sources.search_params("platsbanken", "data", region="zYpL_aGk_QY8") # Midland
|
||||
with pytest.raises(ValueError, match="provider"):
|
||||
sources.search_params("indeed", "data")
|
||||
Reference in new issue
Block a user