"""The `search` plugin, `country_of`, and what the pull and the CLI do with searches. No network: Jobindex's feed is a trimmed copy of a real one (2026-09-23), Adzuna's is written from its documented response.""" from __future__ import annotations import sys from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, ClassVar import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 from plugins import search # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app import cli # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 from app.models import country_of # noqa: E402 from app.storage import Store # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "search" @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") # ----------------------------------------------------------------- country @pytest.mark.parametrize(("place", "code"), [ ("Berlin, Germany", "DE"), ("København K", "DK"), ("Kongens Lyngby", "DK"), ("Boston, MA", "US"), ("Berlin, DE", "DE"), # a state code only when nothing else matched ("Remote - US", "US"), ("London | Remote", "GB"), ("Northern Ireland", "GB"), # the longer name wins over "ireland" ("Remote - Europe", "EU"), ("Home based - EMEA", "EU"), ("London, EMEA", "GB"), # a country beats the region it is in ("London or New York", ""), ("Remote", ""), ("", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None: def item(source_id: str, **raw: Any) -> Item: it = Item(source_id=source_id, location="https://x.example/1", title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw) it.source = "ats" return it store.upsert_role(item("a", place="Copenhagen, Denmark")) store.upsert_role(item("b", place="Remote", country="EU")) store.upsert_role(item("c", place="Somewhere")) assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} # ----------------------------------------------------------------- parsers def test_jobindex_reads_title_company_town_and_date() -> None: rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes()) assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"] paid, bare, robot = rows # A paid ad: the company is its logo's alt text, the title its

. assert paid["company"] == "DSV" assert paid["title"].startswith("Dangerous Goods Compliance Manager") assert paid["place"] == "Hedehusene" assert paid["country"] == "DK" assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23" assert paid["description"].startswith("Do you want to minimise") # A crawled ad with no company after the title's comma has none. assert bare["company"] == "" assert bare["place"] == "København" # A crawled ad names its company after the title. assert robot["title"] == "Marketing Content Creator" assert robot["company"] == "Clerk.IO ApS" def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None: assert search._remote("Birkerød, Hybrid position") == "hybrid" assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid" assert search._remote("Remote, Danmark") == "remote" assert search._remote("Hedehusene") is None def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None: rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de") assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused first, predicted = rows assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote" assert first["description"] == "Build our backend services in Python & Go." assert first["company"] == "Acme GmbH" assert first["place"] == "Berlin, Deutschland" assert first["country"] == "DE" assert first["remote"] == "remote" assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR") assert (predicted["salary_min"], predicted["currency"]) == (None, "") def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("ADZUNA_APP_ID", raising=False) monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) with pytest.raises(ValueError, match="app_id"): search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None: pages: list[int] = [] full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)] def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: pages.append(params["page"]) return b"short" if params["page"] == 3 else b"full" monkeypatch.setattr(search, "_get", fake_get) monkeypatch.setattr(search, "parse_jobindex", lambda body: full[:5] if body == b"short" else full) rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type] assert (pages, len(rows), complete) == ([1, 2, 3], 45, True) pages.clear() rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] assert (pages, complete) == ([1, 2], False) def test_provider_of() -> None: assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("adzuna:berlin") == "adzuna" assert search.provider_of("indeed:x") == "" assert search.provider_of("jobindex:") == "" # ------------------------------------------------------------------- pull class FakeConfig: sources: ClassVar[dict[str, object]] = {} searches: ClassVar[list[SearchSeed]] = [] path = None def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None: def fake(client: object, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København", "country": "DK", "remote": None, "url": f"https://x.example/{i}", "posted": None, "description": "", "salary_min": None, "salary_max": None, "currency": ""} for i in ids], complete monkeypatch.setitem(search._SEARCHES, "jobindex", fake) def _pull(store: Store) -> dict[str, Any]: batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] for batch in batches: pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] class Tally: results: ClassVar[list[dict[str, Any]]] = [] return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type] def test_a_complete_search_closes_what_it_stopped_listing( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend", "area": 1}) _stub_search(monkeypatch, ["h1", "h2"], complete=True) _pull(store) roles = {r.source_id: r for r in store.roles()} assert set(roles) == {"jobindex:h1", "jobindex:h2"} assert roles["jobindex:h1"].country == "DK" assert roles["jobindex:h1"].subscription_id is not None _stub_search(monkeypatch, ["h2"], complete=True) result = _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h2"} assert (result["searches"], result["closed"]) == (1, 1) def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend"}) _stub_search(monkeypatch, ["h1", "h2"], complete=False) _pull(store) _stub_search(monkeypatch, ["h2"], complete=False) _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"} old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() with store.transaction() as conn: conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,)) _pull(store) assert {r.source_id for r in store.roles()} == {"jobindex:h2"} def test_seed_searches_imports_once(store: Store) -> None: config = FakeConfig() config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex", params={"q": "backend", "area": 1})] assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type] sub = store.subscriptions(source="search")[0] assert (sub.target, sub.params) == ("jobindex:copenhagen-backend", {"q": "backend", "area": 1}) store.unsubscribe(sub.id) with store.transaction() as conn: conn.execute("DELETE FROM subscriptions") assert pull.seed_searches(config, store) == [] # type: ignore[arg-type] # -------------------------------------------------------------------- cli def test_prune_unsubscribes_boards_with_no_role_in_the_region( store: Store, capsys: pytest.CaptureFixture[str]) -> None: def board(slug: str, *places: str) -> int: sub = store.add_subscription("ats", f"greenhouse:{slug}", label=slug, company=slug, board="greenhouse") for n, place in enumerate(places): it = Item(source_id=f"greenhouse:{slug}:{n}", location="https://x.example/1", title=f"Engineer {n}", body="", raw={"company": slug, "place": place}) it.source = "ats" store.upsert_role(it, sub) return sub.id us = board("usonly", "Boston, MA", "Remote - US") both = board("both", "San Francisco, CA", "London, UK") vague = board("vague", "Remote", "Hybrid") cli.prune_companies(store, "europe", apply=False) assert "usonly" in capsys.readouterr().out assert len(store.subscriptions(source="ats", fetchable=True)) == 3 cli.prune_companies(store, "europe", apply=True) fetchable = {s.id for s in store.subscriptions(source="ats", fetchable=True)} assert fetchable == {both, vague} assert all(r.subscription_id != us for r in store.roles()) cli.prune_companies(store, "DK,EU", apply=True) assert {s.id for s in store.subscriptions(source="ats", fetchable=True)} == {vague} def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None: _stub_search(monkeypatch, ["h1"], complete=True) source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", "params": {"q": "x"}}]}) items = list(source.fetch()) assert [i.source_id for i in items] == ["jobindex:h1"] assert items[0].raw["search"] == "jobindex:cph" assert source.state["search:complete:jobindex:cph"] == "1" assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'