"""The `search` plugin, `country_of`, and what the pull and the CLI do with searches. No network: Jobindex's feed is a trimmed copy of a real one (2026-09-23), Adzuna's is written from its documented response.""" from __future__ import annotations import json import sys from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, ClassVar import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 from plugins import search # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 from pipeline import Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "search" @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") # ----------------------------------------------------------------- country @pytest.mark.parametrize(("place", "code"), [ ("Berlin, Germany", "DE"), ("København K", "DK"), ("Kongens Lyngby", "DK"), ("Boston, MA", "US"), ("Berlin, DE", "DE"), # a state code only when nothing else matched ("Remote - US", "US"), ("London | Remote", "GB"), ("Northern Ireland", "GB"), # the longer name wins over "ireland" ("Remote - Europe", "EU"), ("Home based - EMEA", "EU"), ("London, EMEA", "GB"), # a country beats the region it is in ("London or New York", ""), ("Remote", ""), ("", ""), ("Wrocław", "PL"), # ł and đ do not decompose under NFKD ("Łódź, Poland", "PL"), ("Bogota, CO", "CO"), # a state code that is a country's too ("Paderborn, DE", ""), # ... is no US state without a US place ("Sliema, MT", ""), ("Casablanca, MA", "MA"), ("Cambridge, MA", "US"), ("Chicago, IL", "US"), ("Madrid, MD, Spain", "ES"), ("Dublin, London", "EU"), # several European countries ("Anywhere in France, Belgium, Spain", "EU"), ("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"), ("Remote, EU or US", ""), # a region and a country outside it ("London, Ontario", "CA"), # a province beats the city before it ("Bellevue, Washington", "US"), ("Tilburg, NB, Netherlands", "NL"), # a country beats a province code ("Atlanta, Georgia", "US"), ("Tbilisi, Georgia", "GE"), ("Malaysia", "MY"), ("Belgrade, Serbia", "RS"), ("Dubai", "AE"), ("United Arab Emirates", "AE"), ("Buenos Aires, , Argentina", "AR"), ("Leicester (GB)", "GB"), # a code in parentheses when nothing else ("Cagliari (CA)", "IT"), # ... and not over a name ("Hybrid", ""), ("Distributed", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code @pytest.mark.parametrize(("place", "mode"), [ ("Remote", "remote"), ("Distributed", "remote"), ("Home based - Worldwide", "remote"), ("Remote - US", "remote"), ("Hybrid", "hybrid"), ("Remote or hybrid, Berlin", "hybrid"), ("In-Office", "onsite"), ("Berlin, Germany", ""), ("", ""), ]) def test_remote_of(place: str, mode: str) -> None: assert remote_of(place) == mode def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None: def item(source_id: str, **raw: Any) -> Item: it = Item(source_id=source_id, location="https://x.example/1", title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw) it.source = "ats" return it store.upsert_role(item("a", place="Copenhagen, Denmark")) store.upsert_role(item("b", place="Remote", country="EU")) store.upsert_role(item("c", place="Somewhere")) assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} # ----------------------------------------------------------------- parsers def test_jobindex_reads_title_company_town_and_date() -> None: rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes()) assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"] paid, bare, robot = rows # A paid ad: the company is its logo's alt text, the title its

. assert paid["company"] == "DSV" assert paid["title"].startswith("Dangerous Goods Compliance Manager") assert paid["place"] == "Hedehusene" assert paid["country"] == "DK" assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23" assert paid["description"].startswith("Do you want to minimise") # A crawled ad with no company after the title's comma has none. assert bare["company"] == "" assert bare["place"] == "København" # A crawled ad names its company after the title. assert robot["title"] == "Marketing Content Creator" assert robot["company"] == "Clerk.IO ApS" def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None: assert search._remote("Birkerød, Hybrid position") == "hybrid" assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid" assert search._remote("Remote, Danmark") == "remote" assert search._remote("Hedehusene") is None def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None: rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de") assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused first, predicted = rows assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote" assert first["description"] == "Build our backend services in Python & Go." assert first["company"] == "Acme GmbH" assert first["place"] == "Berlin, Deutschland" assert first["country"] == "DE" assert first["remote"] == "remote" assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR") assert (predicted["salary_min"], predicted["currency"]) == (None, "") def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("ADZUNA_APP_ID", raising=False) monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) with pytest.raises(ValueError, match="app_id"): search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None: pages: list[int] = [] full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)] def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: pages.append(params["page"]) return b"short" if params["page"] == 3 else b"full" monkeypatch.setattr(search, "_get", fake_get) monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[:5], 5) if body == b"short" else (full, len(full))) rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type] assert (pages, len(rows), complete) == ([1, 2, 3], 45, True) # Stopped by the page limit with every page full: not complete. pages.clear() rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] assert (pages, complete) == ([1, 2], False) def test_a_dropped_jobindex_item_does_not_pass_for_the_end( monkeypatch: pytest.MonkeyPatch) -> None: """A full page whose parser dropped one item still has 20 items, so the search reads on instead of calling itself complete at 19 rows.""" full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)] monkeypatch.setattr(search, "_get", lambda client, url, params: b"") monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full))) _rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] assert not complete def test_a_jobindex_answer_without_a_channel_raises() -> None: with pytest.raises(ValueError, match="channel"): search.parse_jobindex(b"") def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]: """Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`.""" asked: list[int] = [] def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: page = int(url.rsplit("/", 1)[1]) asked.append(page) start = (page - 1) * search.ADZUNA_PAGE results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"} for n in range(start, min(start + search.ADZUNA_PAGE, 60))] body: dict[str, Any] = {"results": results} if count is not None: body["count"] = count return json.dumps(body).encode() monkeypatch.setattr(search, "_get", fake_get) monkeypatch.setenv("ADZUNA_APP_ID", "id") monkeypatch.setenv("ADZUNA_APP_KEY", "key") return asked @pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)]) def test_an_adzuna_search_is_complete_when_it_read_its_count( monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None: _adzuna(monkeypatch, count) rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] assert (len(rows), done) == (60, complete) def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None: asked = _adzuna(monkeypatch, search.ADZUNA_PAGE) rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True) def test_provider_of() -> None: assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("adzuna:berlin") == "adzuna" assert search.provider_of("indeed:x") == "" assert search.provider_of("jobindex:") == "" # ------------------------------------------------------------------- pull class FakeConfig: sources: ClassVar[dict[str, object]] = {} searches: ClassVar[list[SearchSeed]] = [] path = None def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None: def fake(client: object, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København", "country": "DK", "remote": None, "url": f"https://x.example/{i}", "posted": None, "description": "", "salary_min": None, "salary_max": None, "currency": ""} for i in ids], complete monkeypatch.setitem(search._SEARCHES, "jobindex", fake) def _pull(store: Store) -> dict[str, Any]: results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] def test_a_complete_search_closes_what_it_stopped_listing( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend", "area": 1}) _stub_search(monkeypatch, ["h1", "h2"], complete=True) _pull(store) roles = {r.source_id: r for r in store.roles()} assert set(roles) == {"jobindex:h1", "jobindex:h2"} assert roles["jobindex:h1"].country == "DK" assert roles["jobindex:h1"].subscription_id is not None _stub_search(monkeypatch, ["h2"], complete=True) result = _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h2"} assert (result["searches"], result["closed"]) == (1, 1) def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend"}) _stub_search(monkeypatch, ["h1", "h2"], complete=False) _pull(store) _stub_search(monkeypatch, ["h2"], complete=False) _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"} old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() with store.transaction() as conn: conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,)) _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h2"} def test_seed_searches_imports_once(store: Store) -> None: config = FakeConfig() config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex", params={"q": "backend", "area": 1})] assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type] sub = store.subscriptions(source="search")[0] assert (sub.target, sub.params) == ("jobindex:copenhagen-backend", {"q": "backend", "area": 1}) store.unsubscribe(sub.id) with store.transaction() as conn: conn.execute("DELETE FROM subscriptions") assert pull.seed_searches(config, store) == [] # type: ignore[arg-type] # -------------------------------------------------------------------- cli def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None: _stub_search(monkeypatch, ["h1"], complete=True) source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", "params": {"q": "x"}}]}) items = list(source.fetch()) assert [i.source_id for i in items] == ["jobindex:h1"] assert items[0].raw["search"] == "jobindex:cph" assert source.state["search:complete:jobindex:cph"] == "1" assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]' def test_a_search_switched_off_mid_pull_ends_the_pull_closed( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend"}) _stub_search(monkeypatch, ["h1"], complete=False) batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] store.update_subscription(sub.id, subscribed=False, fetch=False) pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type] assert store.roles() == [] # And a batch that starts after the switch reads nothing. assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]