"""The `search` plugin, `country_of`, and what the pull and the CLI do with searches. No network: Jobindex's feed is a trimmed copy of a real one (2026-09-23), Adzuna's is written from its documented response, Platsbanken's is trimmed from a live answer (2026-09-29).""" from __future__ import annotations import json import sys from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, ClassVar import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 from plugins import search # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 from pipeline import Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "search" @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") # ----------------------------------------------------------------- country @pytest.mark.parametrize(("place", "code"), [ ("Berlin, Germany", "DE"), ("København K", "DK"), ("Kongens Lyngby", "DK"), ("Boston, MA", "US"), ("Berlin, DE", "DE"), # a state code only when nothing else matched ("Remote - US", "US"), ("London | Remote", "GB"), ("Northern Ireland", "GB"), # the longer name wins over "ireland" ("Remote - Europe", "EU"), ("Home based - EMEA", "EU"), ("London, EMEA", "GB"), # a country beats the region it is in ("London or New York", ""), ("Remote", ""), ("", ""), ("Wrocław", "PL"), # ł and đ do not decompose under NFKD ("Łódź, Poland", "PL"), ("Bogota, CO", "CO"), # a state code that is a country's too ("Paderborn, DE", ""), # ... is no US state without a US place ("Sliema, MT", ""), ("Casablanca, MA", "MA"), ("Cambridge, MA", "US"), ("Chicago, IL", "US"), ("Madrid, MD, Spain", "ES"), ("Dublin, London", "EU"), # several European countries ("Anywhere in France, Belgium, Spain", "EU"), ("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"), ("Remote, EU or US", ""), # a region and a country outside it ("London, Ontario", "CA"), # a province beats the city before it ("Bellevue, Washington", "US"), ("Tilburg, NB, Netherlands", "NL"), # a country beats a province code ("Atlanta, Georgia", "US"), ("Tbilisi, Georgia", "GE"), ("Malaysia", "MY"), ("Belgrade, Serbia", "RS"), ("Dubai", "AE"), ("United Arab Emirates", "AE"), ("Buenos Aires, , Argentina", "AR"), ("Leicester (GB)", "GB"), # a code in parentheses when nothing else ("Cagliari (CA)", "IT"), # ... and not over a name ("Hybrid", ""), ("Distributed", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code @pytest.mark.parametrize(("place", "mode"), [ ("Remote", "remote"), ("Distributed", "remote"), ("Home based - Worldwide", "remote"), ("Remote - US", "remote"), ("Hybrid", "hybrid"), ("Remote or hybrid, Berlin", "hybrid"), ("In-Office", "onsite"), ("Berlin, Germany", ""), ("", ""), ]) def test_remote_of(place: str, mode: str) -> None: assert remote_of(place) == mode def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None: def item(source_id: str, **raw: Any) -> Item: it = Item(source_id=source_id, location="https://x.example/1", title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw) it.source = "ats" return it store.upsert_role(item("a", place="Copenhagen, Denmark")) store.upsert_role(item("b", place="Remote", country="EU")) store.upsert_role(item("c", place="Somewhere")) assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} # ----------------------------------------------------------------- parsers def test_jobindex_reads_title_company_town_and_date() -> None: rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes()) assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"] paid, bare, robot = rows # A paid ad: the company is its logo's alt text, the title its
The European Spallation Source (ESS)
"}, "employer": {"name": "European Spallation Source Eric"}, "application_details": {"url": "https://europeanspallationsource.se/careers"}, "workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"}, "workplace_address": {"municipality": "Lund", "region": "Skåne län", "region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}}, {"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision", "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504", "publication_date": "2026-09-03T15:46:06", "description": {"text": "Robotics at ESS."}, "employer": {"name": "European Spallation Source Eric"}, "workplace_model": {"concept_id": None, "label": None}, "workplace_address": {"municipality": None, "region": "Skåne län", "region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}}, {"id": "31400001", "headline": "Data scientist, distansarbete möjligt", "webpage_url": "javascript:alert(1)", "workplace_address": {}}, {"id": "31400002", "headline": "Data scientist, distansarbete möjligt", "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002", "employer": {"name": None}, "workplace_address": {}}, ], } def test_platsbanken_reads_title_company_place_and_date() -> None: rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode()) assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused first, no_town, remote = rows assert first["title"] == "Postdoctoral Researcher" assert first["company"] == "European Spallation Source Eric" assert first["place"] == "Lund" assert first["country"] == "SE" assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942" assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09" assert first["description"].startswith("The European Spallation Source") assert first["remote"] is None assert no_town["place"] == "Skåne län" # the region when there is no municipality assert remote["remote"] == "remote" assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None) def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None, stock: int = 250) -> list[dict[str, Any]]: """Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request, stating `total`. Returns the params of each request.""" asked: list[dict[str, Any]] = [] def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: asked.append(params) start = params["offset"] hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}"} for n in range(start, min(start + params["limit"], stock))] body: dict[str, Any] = {"hits": hits} if total is not None: body["total"] = {"value": total} return json.dumps(body).encode() monkeypatch.setattr(search, "_get", fake_get) return asked @pytest.mark.parametrize(("total", "pages", "found", "complete"), [ (250, 5, 250, True), # a short third page ends it (300, 5, 250, False), # the stated total is more than there was (250, 2, 200, False), # stopped by the page limit (None, 5, 250, True), # no total: a short page is the end ]) def test_a_platsbanken_search_is_complete_when_it_read_its_total( monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int, complete: bool) -> None: _platsbanken(monkeypatch, total) rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type] assert (len(rows), done) == (found, complete) def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset( monkeypatch: pytest.MonkeyPatch) -> None: asked = _platsbanken(monkeypatch, 200, stock=200) rows, done = search.search_platsbanken( None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type] assert (len(rows), done) == (200, True) assert [p["offset"] for p in asked] == [0, 100] assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU" assert asked[0]["limit"] == search.PLATSBANKEN_PAGE def test_a_platsbanken_search_without_a_region_sends_none( monkeypatch: pytest.MonkeyPatch) -> None: asked = _platsbanken(monkeypatch, 5, stock=5) search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type] assert "region" not in asked[0] def test_a_platsbanken_search_never_asks_past_the_offset_limit( monkeypatch: pytest.MonkeyPatch) -> None: asked = _platsbanken(monkeypatch, 5000, stock=5000) monkeypatch.setattr(search, "MAX_PAGES", 30) _rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type] assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET assert not done def test_a_platsbanken_search_refuses_an_unknown_region() -> None: with pytest.raises(ValueError, match="region"): search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type] def test_platsbanken_regions_are_the_21_lan() -> None: assert len(search.PLATSBANKEN_REGIONS) == 21 assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län" def test_provider_of() -> None: assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("adzuna:berlin") == "adzuna" assert search.provider_of("platsbanken:skane-data") == "platsbanken" assert search.provider_of("indeed:x") == "" assert search.provider_of("jobindex:") == "" # ------------------------------------------------------------------- pull class FakeConfig: sources: ClassVar[dict[str, object]] = {} searches: ClassVar[list[SearchSeed]] = [] path = None def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None: def fake(client: object, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København", "country": "DK", "remote": None, "url": f"https://x.example/{i}", "posted": None, "description": "", "salary_min": None, "salary_max": None, "currency": ""} for i in ids], complete monkeypatch.setitem(search._SEARCHES, "jobindex", fake) def _pull(store: Store) -> dict[str, Any]: results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] def test_a_complete_search_closes_what_it_stopped_listing( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend", "area": 1}) _stub_search(monkeypatch, ["h1", "h2"], complete=True) _pull(store) roles = {r.source_id: r for r in store.roles()} assert set(roles) == {"jobindex:h1", "jobindex:h2"} assert roles["jobindex:h1"].country == "DK" assert roles["jobindex:h1"].subscription_id is not None _stub_search(monkeypatch, ["h2"], complete=True) result = _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h2"} assert (result["searches"], result["closed"]) == (1, 1) def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend"}) _stub_search(monkeypatch, ["h1", "h2"], complete=False) _pull(store) _stub_search(monkeypatch, ["h2"], complete=False) _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"} old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() with store.transaction() as conn: conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,)) _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h2"} def test_seed_searches_imports_once(store: Store) -> None: config = FakeConfig() config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex", params={"q": "backend", "area": 1})] assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type] sub = store.subscriptions(source="search")[0] assert (sub.target, sub.params) == ("jobindex:copenhagen-backend", {"q": "backend", "area": 1}) store.unsubscribe(sub.id) with store.transaction() as conn: conn.execute("DELETE FROM subscriptions") assert pull.seed_searches(config, store) == [] # type: ignore[arg-type] # -------------------------------------------------------------------- cli def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None: _stub_search(monkeypatch, ["h1"], complete=True) source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", "params": {"q": "x"}}]}) items = list(source.fetch()) assert [i.source_id for i in items] == ["jobindex:h1"] assert items[0].raw["search"] == "jobindex:cph" assert source.state["search:complete:jobindex:cph"] == "1" assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]' def test_a_search_switched_off_mid_pull_ends_the_pull_closed( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend"}) _stub_search(monkeypatch, ["h1"], complete=False) batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] store.update_subscription(sub.id, subscribed=False, fetch=False) pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type] assert store.roles() == [] # And a batch that starts after the switch reads nothing. assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]