"""The `search` plugin over the runner and the provider readers, `country_of`, and what the pull does with searches. No network: `_runner.fetch` is a stub, and a fake provider stands in where the pull is what is tested. The readers' own parsing is tested in `test_readers_.py`.""" from __future__ import annotations import json import sys import types from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, ClassVar from urllib.parse import parse_qs, urlsplit import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 from app import prep # noqa: E402 from plugins import readers, search # noqa: E402 from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 from pipeline import RunContext, Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 from prepped import upsert # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "search" @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") # ----------------------------------------------------------------- country @pytest.mark.parametrize(("place", "code"), [ ("Berlin, Germany", "DE"), ("København K", "DK"), ("Kongens Lyngby", "DK"), ("Boston, MA", "US"), ("Berlin, DE", "DE"), # a state code only when nothing else matched ("Remote - US", "US"), ("London | Remote", "GB"), ("Northern Ireland", "GB"), # the longer name wins over "ireland" ("Remote - Europe", "EU"), ("Home based - EMEA", "EU"), ("London, EMEA", "GB"), # a country beats the region it is in ("London or New York", ""), ("Remote", ""), ("", ""), ("Wrocław", "PL"), # ł and đ do not decompose under NFKD ("Łódź, Poland", "PL"), ("Bogota, CO", "CO"), # a state code that is a country's too ("Ellwangen, DE", ""), # ... is no US state without a US place ("Sliema, MT", ""), ("Casablanca, MA", "MA"), ("Cambridge, MA", "US"), ("Chicago, IL", "US"), ("Madrid, MD, Spain", "ES"), ("Dublin, London", "EU"), # several European countries ("Anywhere in France, Belgium, Spain", "EU"), ("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"), ("Remote, EU or US", ""), # a region and a country outside it ("London, Ontario", "CA"), # a province beats the city before it ("Bellevue, Washington", "US"), ("Tilburg, NB, Netherlands", "NL"), # a country beats a province code ("Atlanta, Georgia", "US"), ("Tbilisi, Georgia", "GE"), ("Malaysia", "MY"), ("Belgrade, Serbia", "RS"), ("Dubai", "AE"), ("United Arab Emirates", "AE"), ("Buenos Aires, , Argentina", "AR"), ("Leicester (GB)", "GB"), # a code in parentheses when nothing else ("Cagliari (CA)", "IT"), # ... and not over a name ("Hybrid", ""), ("Distributed", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code @pytest.mark.parametrize(("place", "mode"), [ ("Remote", "remote"), ("Distributed", "remote"), ("Home based - Worldwide", "remote"), ("Remote - US", "remote"), ("Hybrid", "hybrid"), ("Remote or hybrid, Berlin", "hybrid"), ("In-Office", "onsite"), ("Berlin, Germany", ""), ("", ""), ]) def test_remote_of(place: str, mode: str) -> None: assert remote_of(place) == mode def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None: def item(source_id: str, **raw: Any) -> Item: it = Item(source_id=source_id, location="https://x.example/1", title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw) it.source = "ats" return it upsert(store, item("a", place="Copenhagen, Denmark")) upsert(store, item("b", place="Remote", country="EU")) upsert(store, item("c", place="Somewhere")) assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} # ----------------------------------------------------- read through the runner def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]: """`_runner.fetch` answering `answer(url)`, remembering the URLs asked.""" asked: list[str] = [] def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes: assert allow(urlsplit(url).hostname), url asked.append(url) return answer(url) monkeypatch.setattr(_runner, "fetch", fake_fetch) monkeypatch.setattr(search, "PAGE_DELAY", 0) return asked def _jobindex_page(url: str) -> bytes: """A full Jobindex page (20 items) whatever page is asked.""" page = int(parse_qs(urlsplit(url).query)["page"][0]) items = "".join(f"Role {page}-{n}, Acme" f"https://www.jobindex.dk/vis-job/r{page}x{n}" f"<p>Teaser {n}</p>" for n in range(20)) return f"Jobindex{items}".encode() @pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)]) def test_the_saved_pages_param_bounds_the_pages_read( monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None: urls = _web(monkeypatch, _jobindex_page) params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else {"pages": pages}) rows, complete = search.run_search("jobindex:cph", params) assert len(urls) == asked and len(rows) == 20 * asked assert not complete # Jobindex states no total: never complete query = parse_qs(urlsplit(urls[0]).query) assert (query["q"], query["geoareaid"]) == (["python"], ["1"]) def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body( monkeypatch: pytest.MonkeyPatch) -> None: _web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes()) source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", "params": {"q": "x", "pages": 1}}]}) items = list(source.fetch()) paid = items[0] assert paid.source_id == "jobindex:h1701165" assert (paid.body, paid.excerpt) == ("", "") assert paid.raw["teaser"].startswith("Do you want to minimise") assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK" assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}] assert paid.raw["place"] == "Hedehusene" assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph") assert source.state["search:complete:jobindex:cph"] == "" def _adzuna(count: int | None, stock: int = 60) -> Any: def answer(url: str) -> bytes: page = int(urlsplit(url).path.rsplit("/", 1)[1]) start = (page - 1) * 50 results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"} for n in range(start, min(start + 50, stock))] body: dict[str, Any] = {"results": results} if count is not None: body["count"] = count return json.dumps(body).encode() return answer def test_a_search_without_its_key_is_refused_before_any_request( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("ADZUNA_APP_ID", raising=False) monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) urls = _web(monkeypatch, _adzuna(60)) with pytest.raises(ValueError, match="app_id"): search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {}) assert urls == [] def test_the_key_comes_from_the_config_before_the_environment( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("ADZUNA_APP_ID", "env-id") monkeypatch.setenv("ADZUNA_APP_KEY", "env-key") urls = _web(monkeypatch, _adzuna(60)) search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {"adzuna": {"app_id": "cfg-id"}}) query = parse_qs(urlsplit(urls[0]).query) assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"]) assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1" @pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)]) def test_an_adzuna_search_is_complete_when_it_read_its_count( monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None: _web(monkeypatch, _adzuna(count)) rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {"adzuna": {"app_id": "i", "app_key": "k"}}) assert (len(rows), done) == (60, complete) assert rows[0]["place"] == "" def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total( monkeypatch: pytest.MonkeyPatch) -> None: def answer(url: str) -> bytes: query = parse_qs(urlsplit(url).query) start = int(query["offset"][0]) hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}", "workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]} for n in range(start, min(start + 100, 200))] return json.dumps({"total": {"value": 200}, "hits": hits}).encode() urls = _web(monkeypatch, answer) rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"}) assert (len(rows), done) == (200, True) offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls] assert offsets == [["0"], ["100"]] assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"] assert rows[0]["place"] == "Lund; Malmö" def test_an_unknown_provider_is_refused() -> None: with pytest.raises(ValueError, match="provider"): search.run_search("indeed:x", {"q": "x"}) def test_provider_of() -> None: assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("adzuna:berlin") == "adzuna" assert search.provider_of("platsbanken:skane-data") == "platsbanken" assert search.provider_of("indeed:x") == "" assert search.provider_of("jobindex:") == "" # ------------------------------------------------------------------- pull class FakeConfig: sources: ClassVar[dict[str, object]] = {} searches: ClassVar[list[SearchSeed]] = [] path = None def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None: """A provider `fake` that lists `ids` in one page and states a total of as many, or of one more when the search is to be incomplete.""" module: Any = types.ModuleType("fake") module.NAME, module.LABEL = "fake", "Fake" module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated", places="one", fields=Fields(country="stated"), robots="test (checked 2026-09-30)") module.allowed = lambda slug: lambda host: host == "search.fake.example" module.PARAMS = {"q": "query"} module.clean_params = lambda params: dict(params) def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: assert ask.params["q"] return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None module.page = page module.rows = lambda decoded, slug: [ {"source_id": i, "title": f"Engineer {i}", "company": "Acme", "url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}], "country": "DK", "remote": None, "posted": None, "description": ""} for i in decoded] assert not readers.problems(module, "fake") monkeypatch.setitem(readers.PROVIDERS, "fake", module) def _pull(store: Store) -> dict[str, Any]: results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] prep.run(store) return closed def test_a_complete_search_closes_nothing_by_its_list_only_by_age( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: """A search is never a company's whole list: one that read as many results as its provider stated is stamped complete, and a role it stopped listing still stays open until no search has listed it for `STALE_DAYS`.""" store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1", "h2"], complete=True) _pull(store) roles = {r.source_id: r for r in store.roles()} assert set(roles) == {"fake:h1", "fake:h2"} assert roles["fake:h1"].country == "DK" assert roles["fake:h1"].subscription_id is not None _fake_provider(monkeypatch, ["h2"], complete=True) result = _pull(store) assert store.state["search:complete:fake:cph"] == "1" assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"} assert (result["searches"], result["closed"]) == (1, 0) old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() with store.transaction() as conn: conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,)) result = _pull(store) assert {r.source_id for r in store.roles()} == {"fake:h2"} assert (result["searches"], result["closed"]) == (1, 1) def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: """Fifteen of twenty roles dropping out of a complete search closed them, and named the search as closing many; now they stay open and the settle has nothing to say.""" store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) twenty = [f"h{n}" for n in range(20)] _fake_provider(monkeypatch, twenty, complete=True) _pull(store) _fake_provider(monkeypatch, twenty[:5], complete=True) said: list[object] = [] results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type] context=RunContext(on_notice=said.append)) assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0) assert len(store.roles()) == 20 assert said == [] def test_an_incomplete_search_closes_nothing_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1", "h2"], complete=False) _pull(store) _fake_provider(monkeypatch, ["h2"], complete=False) _pull(store) assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"} old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() with store.transaction() as conn: conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,)) _pull(store) assert {r.source_id for r in store.roles()} == {"fake:h2"} def test_seed_searches_imports_once(store: Store) -> None: config = FakeConfig() config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex", params={"q": "backend", "area": 1})] assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type] sub = store.subscriptions(source="search")[0] assert (sub.target, sub.params) == ("jobindex:copenhagen-backend", {"q": "backend", "area": 1}) store.unsubscribe(sub.id) with store.transaction() as conn: conn.execute("DELETE FROM subscriptions") assert pull.seed_searches(config, store) == [] # type: ignore[arg-type] def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None: _fake_provider(monkeypatch, ["h1"], complete=True) source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen", "params": {"q": "x"}}]}) items = list(source.fetch()) assert [i.source_id for i in items] == ["fake:h1"] assert items[0].raw["search"] == "fake:cph" assert source.state["search:complete:fake:cph"] == "1" assert source.state["search:ids:fake:cph"] == '["fake:h1"]' def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count( monkeypatch: pytest.MonkeyPatch) -> None: """The search driver reads through the same listing code as a board: the runner drops the row the provider handed over without a title, and the read is still complete, by the distinct result ids.""" _fake_provider(monkeypatch, ["h1", "h2"], complete=True) module = readers.PROVIDERS["fake"] handed = module.rows module.rows = lambda decoded, slug: [ row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)] source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen", "params": {"q": "x"}}]}) assert [i.source_id for i in source.fetch()] == ["fake:h1"] assert source.state["search:complete:fake:cph"] == "1" assert source.state["search:dropped:fake:cph"] == "1" def test_a_search_that_fails_is_skipped_and_stamps_nothing( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("ADZUNA_APP_ID", raising=False) monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin", "params": {"q": "x", "country": "de"}}]}) assert list(source.fetch()) == [] assert "search:ok:adzuna:berlin" not in source.state def test_a_search_switched_off_mid_pull_ends_the_pull_closed( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: sub = store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1"], complete=False) batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] store.update_subscription(sub.id, subscribed=False, fetch=False) pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type] assert store.roles() == [] # And a batch that starts after the switch reads nothing. assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type] # ------------------------------------------------- failures and the guard def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None: """The provider `fake` of `_fake_provider`, answering with an error.""" _fake_provider(monkeypatch, [], complete=True) def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: raise ValueError("hits renamed") readers.PROVIDERS["fake"].page = page def test_a_search_whose_provider_raises_is_recorded_failing( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1"], complete=True) _pull(store) _broken_provider(monkeypatch) result = _pull(store) assert result["unanswered"] == 1 since, reason = pull.failing_boards(store)["fake:cph"] assert since and reason == "ValueError: hits renamed" assert {r.source_id for r in store.roles()} == {"fake:h1"} # Failing for `STALE_DAYS` closes its roles, as a board's. old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat( timespec="seconds") store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason}) _pull(store) assert store.roles() == [] # Its next answer removes the record and reopens what it lists. _fake_provider(monkeypatch, ["h1"], complete=True) _pull(store) assert pull.failing_boards(store) == {} assert {r.source_id for r in store.roles()} == {"fake:h1"} def test_a_search_that_lists_nothing_after_forty_is_suspect( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) forty = [f"h{n}" for n in range(40)] _fake_provider(monkeypatch, forty, complete=True) _pull(store) _fake_provider(monkeypatch, [], complete=True) result = _pull(store) assert (result["unanswered"], result["closed"]) == (1, 0) assert len(store.roles()) == 40 assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]