"""The `search` plugin over the runner and the provider readers, `country_of`, and what the pull does with searches. No network: `_runner.fetch` is a stub, and a fake provider stands in where the pull is what is tested. The readers' own parsing is tested in `test_readers_.py`.""" from __future__ import annotations import json import sys import types from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, ClassVar from urllib.parse import parse_qs, urlsplit import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 from app import prep # noqa: E402 from plugins import readers, search # noqa: E402 from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 from pipeline import Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 from prepped import upsert # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "search" @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") # ----------------------------------------------------------------- country @pytest.mark.parametrize(("place", "code"), [ ("Berlin, Germany", "DE"), ("København K", "DK"), ("Kongens Lyngby", "DK"), ("Boston, MA", "US"), ("Berlin, DE", "DE"), # a state code only when nothing else matched ("Remote - US", "US"), ("London | Remote", "GB"), ("Northern Ireland", "GB"), # the longer name wins over "ireland" ("Remote - Europe", "EU"), ("Home based - EMEA", "EU"), ("London, EMEA", "GB"), # a country beats the region it is in ("London or New York", ""), ("Remote", ""), ("", ""), ("Wrocław", "PL"), # ł and đ do not decompose under NFKD ("Łódź, Poland", "PL"), ("Bogota, CO", "CO"), # a state code that is a country's too ("Paderborn, DE", ""), # ... is no US state without a US place ("Sliema, MT", ""), ("Casablanca, MA", "MA"), ("Cambridge, MA", "US"), ("Chicago, IL", "US"), ("Madrid, MD, Spain", "ES"), ("Dublin, London", "EU"), # several European countries ("Anywhere in France, Belgium, Spain", "EU"), ("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"), ("Remote, EU or US", ""), # a region and a country outside it ("London, Ontario", "CA"), # a province beats the city before it ("Bellevue, Washington", "US"), ("Tilburg, NB, Netherlands", "NL"), # a country beats a province code ("Atlanta, Georgia", "US"), ("Tbilisi, Georgia", "GE"), ("Malaysia", "MY"), ("Belgrade, Serbia", "RS"), ("Dubai", "AE"), ("United Arab Emirates", "AE"), ("Buenos Aires, , Argentina", "AR"), ("Leicester (GB)", "GB"), # a code in parentheses when nothing else ("Cagliari (CA)", "IT"), # ... and not over a name ("Hybrid", ""), ("Distributed", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code @pytest.mark.parametrize(("place", "mode"), [ ("Remote", "remote"), ("Distributed", "remote"), ("Home based - Worldwide", "remote"), ("Remote - US", "remote"), ("Hybrid", "hybrid"), ("Remote or hybrid, Berlin", "hybrid"), ("In-Office", "onsite"), ("Berlin, Germany", ""), ("", ""), ]) def test_remote_of(place: str, mode: str) -> None: assert remote_of(place) == mode def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None: def item(source_id: str, **raw: Any) -> Item: it = Item(source_id=source_id, location="https://x.example/1", title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw) it.source = "ats" return it upsert(store, item("a", place="Copenhagen, Denmark")) upsert(store, item("b", place="Remote", country="EU")) upsert(store, item("c", place="Somewhere")) assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} # ----------------------------------------------------- read through the runner def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]: """`_runner.fetch` answering `answer(url)`, remembering the URLs asked.""" asked: list[str] = [] def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes: assert allow(urlsplit(url).hostname), url asked.append(url) return answer(url) monkeypatch.setattr(_runner, "fetch", fake_fetch) monkeypatch.setattr(search, "PAGE_DELAY", 0) return asked def _jobindex_page(url: str) -> bytes: """A full Jobindex page (20 items) whatever page is asked.""" page = int(parse_qs(urlsplit(url).query)["page"][0]) items = "".join(f"Role {page}-{n}, Acme" f"https://www.jobindex.dk/vis-job/r{page}x{n}" f"<p>Teaser {n}</p>" for n in range(20)) return f"Jobindex{items}".encode() @pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)]) def test_the_saved_pages_param_bounds_the_pages_read( monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None: urls = _web(monkeypatch, _jobindex_page) params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else {"pages": pages}) rows, complete = search.run_search("jobindex:cph", params) assert len(urls) == asked and len(rows) == 20 * asked assert not complete # Jobindex states no total: never complete query = parse_qs(urlsplit(urls[0]).query) assert (query["q"], query["geoareaid"]) == (["python"], ["1"]) def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body( monkeypatch: pytest.MonkeyPatch) -> None: _web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes()) source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", "params": {"q": "x", "pages": 1}}]}) items = list(source.fetch()) paid = items[0] assert paid.source_id == "jobindex:h1701165" assert (paid.body, paid.excerpt) == ("", "") assert paid.raw["teaser"].startswith("Do you want to minimise") assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK" assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}] assert paid.raw["place"] == "Hedehusene" assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph") assert source.state["search:complete:jobindex:cph"] == "" def _adzuna(count: int | None, stock: int = 60) -> Any: def answer(url: str) -> bytes: page = int(urlsplit(url).path.rsplit("/", 1)[1]) start = (page - 1) * 50 results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"} for n in range(start, min(start + 50, stock))] body: dict[str, Any] = {"results": results} if count is not None: body["count"] = count return json.dumps(body).encode() return answer def test_a_search_without_its_key_is_refused_before_any_request( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("ADZUNA_APP_ID", raising=False) monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) urls = _web(monkeypatch, _adzuna(60)) with pytest.raises(ValueError, match="app_id"): search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {}) assert urls == [] def test_the_key_comes_from_the_config_before_the_environment( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("ADZUNA_APP_ID", "env-id") monkeypatch.setenv("ADZUNA_APP_KEY", "env-key") urls = _web(monkeypatch, _adzuna(60)) search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {"adzuna": {"app_id": "cfg-id"}}) query = parse_qs(urlsplit(urls[0]).query) assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"]) assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1" @pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)]) def test_an_adzuna_search_is_complete_when_it_read_its_count( monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None: _web(monkeypatch, _adzuna(count)) rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {"adzuna": {"app_id": "i", "app_key": "k"}}) assert (len(rows), done) == (60, complete) assert rows[0]["place"] == "" def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total( monkeypatch: pytest.MonkeyPatch) -> None: def answer(url: str) -> bytes: query = parse_qs(urlsplit(url).query) start = int(query["offset"][0]) hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}", "workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]} for n in range(start, min(start + 100, 200))] return json.dumps({"total": {"value": 200}, "hits": hits}).encode() urls = _web(monkeypatch, answer) rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"}) assert (len(rows), done) == (200, True) offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls] assert offsets == [["0"], ["100"]] assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"] assert rows[0]["place"] == "Lund; Malmö" def test_an_unknown_provider_is_refused() -> None: with pytest.raises(ValueError, match="provider"): search.run_search("indeed:x", {"q": "x"}) def test_provider_of() -> None: assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("adzuna:berlin") == "adzuna" assert search.provider_of("platsbanken:skane-data") == "platsbanken" assert search.provider_of("indeed:x") == "" assert search.provider_of("jobindex:") == "" # ------------------------------------------------------------------- pull class FakeConfig: sources: ClassVar[dict[str, object]] = {} searches: ClassVar[list[SearchSeed]] = [] path = None def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None: """A provider `fake` that lists `ids` in one page and states a total of as many, or of one more when the search is to be incomplete.""" module: Any = types.ModuleType("fake") module.NAME, module.LABEL = "fake", "Fake" module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated", places="one", fields=Fields(country="stated"), robots="test") module.allowed = lambda slug: lambda host: host == "search.fake.example" def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: assert ask.params["q"] return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None module.page = page module.rows = lambda decoded, slug: [ {"source_id": i, "title": f"Engineer {i}", "company": "Acme", "url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}], "country": "DK", "remote": None, "posted": None, "description": ""} for i in decoded] assert not readers.problems(module, "fake") monkeypatch.setitem(readers.PROVIDERS, "fake", module) def _pull(store: Store) -> dict[str, Any]: results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] prep.run(store) return closed def test_a_complete_search_closes_what_it_stopped_listing( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1", "h2"], complete=True) _pull(store) roles = {r.source_id: r for r in store.roles()} assert set(roles) == {"fake:h1", "fake:h2"} assert roles["fake:h1"].country == "DK" assert roles["fake:h1"].subscription_id is not None _fake_provider(monkeypatch, ["h2"], complete=True) result = _pull(store) assert {r.source_id for r in store.roles()} == {"fake:h2"} assert (result["searches"], result["closed"]) == (1, 1) def test_an_incomplete_search_closes_nothing_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1", "h2"], complete=False) _pull(store) _fake_provider(monkeypatch, ["h2"], complete=False) _pull(store) assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"} old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() with store.transaction() as conn: conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,)) _pull(store) assert {r.source_id for r in store.roles()} == {"fake:h2"} def test_seed_searches_imports_once(store: Store) -> None: config = FakeConfig() config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex", params={"q": "backend", "area": 1})] assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type] sub = store.subscriptions(source="search")[0] assert (sub.target, sub.params) == ("jobindex:copenhagen-backend", {"q": "backend", "area": 1}) store.unsubscribe(sub.id) with store.transaction() as conn: conn.execute("DELETE FROM subscriptions") assert pull.seed_searches(config, store) == [] # type: ignore[arg-type] def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None: _fake_provider(monkeypatch, ["h1"], complete=True) source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen", "params": {"q": "x"}}]}) items = list(source.fetch()) assert [i.source_id for i in items] == ["fake:h1"] assert items[0].raw["search"] == "fake:cph" assert source.state["search:complete:fake:cph"] == "1" assert source.state["search:ids:fake:cph"] == '["fake:h1"]' def test_a_search_that_fails_is_skipped_and_stamps_nothing( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("ADZUNA_APP_ID", raising=False) monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin", "params": {"q": "x", "country": "de"}}]}) assert list(source.fetch()) == [] assert "search:ok:adzuna:berlin" not in source.state def test_a_search_switched_off_mid_pull_ends_the_pull_closed( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: sub = store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1"], complete=False) batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] store.update_subscription(sub.id, subscribed=False, fetch=False) pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type] assert store.roles() == [] # And a batch that starts after the switch reads nothing. assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]