Search providers as reader modules: jobindex, adzuna, platsbanken
Each declares READS with scope=search, keeps every place and hands the excerpt over as teaser. search_params and PROVIDERS follow the declarations. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
b78c9dfc67
14 files changed
+1202
-17
No files matched your search
@@ -0,0 +1,111 @@
|
||||
"""The Platsbanken reader (JobTech search API): 100 hits a page, offsets to
|
||||
2000. No network: `get` is a stub."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, platsbanken # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
|
||||
|
||||
|
||||
def _stock(total: int | None, stock: int, asked: list[str]) -> Any:
|
||||
def get(url: str, **kwargs: Any) -> bytes:
|
||||
asked.append(url)
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
start, limit = int(query["offset"][0]), int(query["limit"][0])
|
||||
body: dict[str, Any] = {"hits": [
|
||||
{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + limit, stock))]}
|
||||
if total is not None:
|
||||
body["total"] = {"value": total}
|
||||
return json.dumps(body).encode()
|
||||
return get
|
||||
|
||||
|
||||
def test_it_is_a_search_reader_with_a_404_robots_txt_recorded() -> None:
|
||||
assert readers.PROVIDERS["platsbanken"] is platsbanken
|
||||
reads = platsbanken.READS
|
||||
assert (reads.scope, reads.page_size, reads.ceiling, reads.places) == (
|
||||
"search", 100, 2000, "many")
|
||||
assert "404" in reads.robots and "2026-09-30" in reads.robots
|
||||
|
||||
|
||||
def test_it_keeps_every_workplace_of_an_ad() -> None:
|
||||
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||
rows = platsbanken.rows(decoded, "")
|
||||
assert total == 32
|
||||
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
||||
first, no_town, bare = rows
|
||||
assert [p["text"] for p in first["places"]] == [
|
||||
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
|
||||
assert first["location"] == "Lund"
|
||||
assert no_town["places"] == [{"text": "Skåne län", "remote": None}]
|
||||
assert bare["places"] == []
|
||||
|
||||
|
||||
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
||||
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||
first, _no_town, remote = platsbanken.rows(decoded, "")
|
||||
assert first["title"] == "Postdoctoral Researcher"
|
||||
assert first["company"] == "European Spallation Source Eric"
|
||||
assert first["country"] == "SE" and first["remote"] is None
|
||||
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
|
||||
assert first["posted"].date().isoformat() == "2026-09-09"
|
||||
assert first["description"].startswith("The European Spallation Source")
|
||||
assert "teaser" not in first
|
||||
assert (remote["remote"], remote["company"], remote["posted"]) == ("remote", "", None)
|
||||
|
||||
|
||||
def test_it_sends_the_region_the_offset_and_no_region_for_all_of_sweden() -> None:
|
||||
asked: list[str] = []
|
||||
get = _stock(250, 250, asked)
|
||||
platsbanken.page(get, "", None, Ask(params={"q": "data", "region": "CaRE_1nn_cSU"}))
|
||||
platsbanken.page(get, "", 100, Ask(params={"q": "data", "region": ""}))
|
||||
assert parse_qs(urlsplit(asked[0]).query) == {
|
||||
"q": ["data"], "sort": ["pubdate-desc"], "limit": ["100"], "offset": ["0"],
|
||||
"region": ["CaRE_1nn_cSU"]}
|
||||
assert "region" not in urlsplit(asked[1]).query and "offset=100" in asked[1]
|
||||
assert urlsplit(asked[0]).netloc == "jobsearch.api.jobtechdev.se"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("cursor", "stock", "after"), [
|
||||
(None, 250, 100), (200, 250, None), # a short page is the end
|
||||
(1900, 2500, 2000), (2000, 2500, None), # the offset ceiling
|
||||
])
|
||||
def test_it_pages_by_offset_up_to_the_ceiling(
|
||||
cursor: int | None, stock: int, after: int | None) -> None:
|
||||
_decoded, _found, _total, nxt = platsbanken.page(
|
||||
_stock(None, stock, []), "", cursor, Ask(params={"q": "x"}))
|
||||
assert nxt == after
|
||||
|
||||
|
||||
def test_it_refuses_an_unknown_region() -> None:
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
platsbanken.page(lambda url, **kw: b"{}", "", None,
|
||||
Ask(params={"q": "x", "region": "zYpL_aGk_QY8"})) # Midland
|
||||
|
||||
|
||||
def test_the_regions_are_the_21_lan() -> None:
|
||||
assert len(platsbanken.REGIONS) == 21
|
||||
assert all(name.endswith(" län") for name in platsbanken.REGIONS.values())
|
||||
|
||||
|
||||
def test_saved_params_keep_a_län_or_none() -> None:
|
||||
assert platsbanken.clean_params({"q": "x", "region": " CaRE_1nn_cSU "}) == {
|
||||
"q": "x", "region": "CaRE_1nn_cSU"}
|
||||
assert platsbanken.clean_params({"q": "x", "region": ""}) == {"q": "x", "region": ""}
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
platsbanken.clean_params({"q": "x", "region": "zYpL_aGk_QY8"})
|
||||
Reference in new issue
Block a user