Files
jobboard/tests/test_search.py
T
WolfandClaude Opus 5.5 3e3060ba50 Link the obvious towns of the Unmapped list: 460 unlinked place texts down to 50
About 190 German towns (mostly 1KOMMA5°'s) and the German states, French,
Italian, Dutch, Asian and other towns, the ISO3 codes and German or French
country names boards write. Towns keep their marks through spelling groups.
The tests that used Paderborn, Jena and Überlingen as unknown towns use
Bopfingen, Lauchheim and Ellwangen instead. Left: names that are no place,
ambiguous ones (Halifax, Richmond, Victoria, Durham, Warwick) and bare codes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 13:34:27 +02:00

374 lines
16 KiB
Python

"""The `search` plugin over the runner and the provider readers, `country_of`,
and what the pull does with searches. No network: `_runner.fetch` is a stub,
and a fake provider stands in where the pull is what is tested. The readers'
own parsing is tested in `test_readers_<provider>.py`."""
from __future__ import annotations
import json
import sys
import types
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, ClassVar
from urllib.parse import parse_qs, urlsplit
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from app import prep # noqa: E402
from plugins import readers, search # noqa: E402
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
# ----------------------------------------------------------------- country
@pytest.mark.parametrize(("place", "code"), [
("Berlin, Germany", "DE"),
("København K", "DK"),
("Kongens Lyngby", "DK"),
("Boston, MA", "US"),
("Berlin, DE", "DE"), # a state code only when nothing else matched
("Remote - US", "US"),
("London | Remote", "GB"),
("Northern Ireland", "GB"), # the longer name wins over "ireland"
("Remote - Europe", "EU"),
("Home based - EMEA", "EU"),
("London, EMEA", "GB"), # a country beats the region it is in
("London or New York", ""),
("Remote", ""),
("", ""),
("Wrocław", "PL"), # ł and đ do not decompose under NFKD
("Łódź, Poland", "PL"),
("Bogota, CO", "CO"), # a state code that is a country's too
("Ellwangen, DE", ""), # ... is no US state without a US place
("Sliema, MT", ""),
("Casablanca, MA", "MA"),
("Cambridge, MA", "US"),
("Chicago, IL", "US"),
("Madrid, MD, Spain", "ES"),
("Dublin, London", "EU"), # several European countries
("Anywhere in France, Belgium, Spain", "EU"),
("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"),
("Remote, EU or US", ""), # a region and a country outside it
("London, Ontario", "CA"), # a province beats the city before it
("Bellevue, Washington", "US"),
("Tilburg, NB, Netherlands", "NL"), # a country beats a province code
("Atlanta, Georgia", "US"),
("Tbilisi, Georgia", "GE"),
("Malaysia", "MY"),
("Belgrade, Serbia", "RS"),
("Dubai", "AE"),
("United Arab Emirates", "AE"),
("Buenos Aires, , Argentina", "AR"),
("Leicester (GB)", "GB"), # a code in parentheses when nothing else
("Cagliari (CA)", "IT"), # ... and not over a name
("Hybrid", ""),
("Distributed", ""),
])
def test_country_of(place: str, code: str) -> None:
assert country_of(place) == code
@pytest.mark.parametrize(("place", "mode"), [
("Remote", "remote"),
("Distributed", "remote"),
("Home based - Worldwide", "remote"),
("Remote - US", "remote"),
("Hybrid", "hybrid"),
("Remote or hybrid, Berlin", "hybrid"),
("In-Office", "onsite"),
("Berlin, Germany", ""),
("", ""),
])
def test_remote_of(place: str, mode: str) -> None:
assert remote_of(place) == mode
def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None:
def item(source_id: str, **raw: Any) -> Item:
it = Item(source_id=source_id, location="https://x.example/1", title="Engineer",
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw)
it.source = "ats"
return it
upsert(store, item("a", place="Copenhagen, Denmark"))
upsert(store, item("b", place="Remote", country="EU"))
upsert(store, item("c", place="Somewhere"))
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
# ----------------------------------------------------- read through the runner
def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]:
"""`_runner.fetch` answering `answer(url)`, remembering the URLs asked."""
asked: list[str] = []
def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes:
assert allow(urlsplit(url).hostname), url
asked.append(url)
return answer(url)
monkeypatch.setattr(_runner, "fetch", fake_fetch)
monkeypatch.setattr(search, "PAGE_DELAY", 0)
return asked
def _jobindex_page(url: str) -> bytes:
"""A full Jobindex page (20 items) whatever page is asked."""
page = int(parse_qs(urlsplit(url).query)["page"][0])
items = "".join(f"<item><title>Role {page}-{n}, Acme</title>"
f"<link>https://www.jobindex.dk/vis-job/r{page}x{n}</link>"
f"<description>&lt;p&gt;Teaser {n}&lt;/p&gt;</description></item>"
for n in range(20))
return f"<rss><channel><title>Jobindex</title>{items}</channel></rss>".encode()
@pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)])
def test_the_saved_pages_param_bounds_the_pages_read(
monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None:
urls = _web(monkeypatch, _jobindex_page)
params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else
{"pages": pages})
rows, complete = search.run_search("jobindex:cph", params)
assert len(urls) == asked and len(rows) == 20 * asked
assert not complete # Jobindex states no total: never complete
query = parse_qs(urlsplit(urls[0]).query)
assert (query["q"], query["geoareaid"]) == (["python"], ["1"])
def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body(
monkeypatch: pytest.MonkeyPatch) -> None:
_web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes())
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
"params": {"q": "x", "pages": 1}}]})
items = list(source.fetch())
paid = items[0]
assert paid.source_id == "jobindex:h1701165"
assert (paid.body, paid.excerpt) == ("", "")
assert paid.raw["teaser"].startswith("Do you want to minimise")
assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK"
assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}]
assert paid.raw["place"] == "Hedehusene"
assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph")
assert source.state["search:complete:jobindex:cph"] == ""
def _adzuna(count: int | None, stock: int = 60) -> Any:
def answer(url: str) -> bytes:
page = int(urlsplit(url).path.rsplit("/", 1)[1])
start = (page - 1) * 50
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
for n in range(start, min(start + 50, stock))]
body: dict[str, Any] = {"results": results}
if count is not None:
body["count"] = count
return json.dumps(body).encode()
return answer
def test_a_search_without_its_key_is_refused_before_any_request(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
urls = _web(monkeypatch, _adzuna(60))
with pytest.raises(ValueError, match="app_id"):
search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {})
assert urls == []
def test_the_key_comes_from_the_config_before_the_environment(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("ADZUNA_APP_ID", "env-id")
monkeypatch.setenv("ADZUNA_APP_KEY", "env-key")
urls = _web(monkeypatch, _adzuna(60))
search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
{"adzuna": {"app_id": "cfg-id"}})
query = parse_qs(urlsplit(urls[0]).query)
assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"])
assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1"
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)])
def test_an_adzuna_search_is_complete_when_it_read_its_count(
monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None:
_web(monkeypatch, _adzuna(count))
rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
{"adzuna": {"app_id": "i", "app_key": "k"}})
assert (len(rows), done) == (60, complete)
assert rows[0]["place"] == ""
def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total(
monkeypatch: pytest.MonkeyPatch) -> None:
def answer(url: str) -> bytes:
query = parse_qs(urlsplit(url).query)
start = int(query["offset"][0])
hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}",
"workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]}
for n in range(start, min(start + 100, 200))]
return json.dumps({"total": {"value": 200}, "hits": hits}).encode()
urls = _web(monkeypatch, answer)
rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"})
assert (len(rows), done) == (200, True)
offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls]
assert offsets == [["0"], ["100"]]
assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"]
assert rows[0]["place"] == "Lund; Malmö"
def test_an_unknown_provider_is_refused() -> None:
with pytest.raises(ValueError, match="provider"):
search.run_search("indeed:x", {"q": "x"})
def test_provider_of() -> None:
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
assert search.provider_of("adzuna:berlin") == "adzuna"
assert search.provider_of("platsbanken:skane-data") == "platsbanken"
assert search.provider_of("indeed:x") == ""
assert search.provider_of("jobindex:") == ""
# ------------------------------------------------------------------- pull
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[SearchSeed]] = []
path = None
def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
"""A provider `fake` that lists `ids` in one page and states a total of
as many, or of one more when the search is to be incomplete."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL = "fake", "Fake"
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
places="one", fields=Fields(country="stated"), robots="test")
module.allowed = lambda slug: lambda host: host == "search.fake.example"
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
assert ask.params["q"]
return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None
module.page = page
module.rows = lambda decoded, slug: [
{"source_id": i, "title": f"Engineer {i}", "company": "Acme",
"url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}],
"country": "DK", "remote": None, "posted": None, "description": ""}
for i in decoded]
assert not readers.problems(module, "fake")
monkeypatch.setitem(readers.PROVIDERS, "fake", module)
def _pull(store: Store) -> dict[str, Any]:
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
prep.run(store)
return closed
def test_a_complete_search_closes_what_it_stopped_listing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
_pull(store)
roles = {r.source_id: r for r in store.roles()}
assert set(roles) == {"fake:h1", "fake:h2"}
assert roles["fake:h1"].country == "DK"
assert roles["fake:h1"].subscription_id is not None
_fake_provider(monkeypatch, ["h2"], complete=True)
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"fake:h2"}
assert (result["searches"], result["closed"]) == (1, 1)
def test_an_incomplete_search_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_fake_provider(monkeypatch, ["h1", "h2"], complete=False)
_pull(store)
_fake_provider(monkeypatch, ["h2"], complete=False)
_pull(store)
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
_pull(store)
assert {r.source_id for r in store.roles()} == {"fake:h2"}
def test_seed_searches_imports_once(store: Store) -> None:
config = FakeConfig()
config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex",
params={"q": "backend", "area": 1})]
assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type]
sub = store.subscriptions(source="search")[0]
assert (sub.target, sub.params) == ("jobindex:copenhagen-backend",
{"q": "backend", "area": 1})
store.unsubscribe(sub.id)
with store.transaction() as conn:
conn.execute("DELETE FROM subscriptions")
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
_fake_provider(monkeypatch, ["h1"], complete=True)
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
"params": {"q": "x"}}]})
items = list(source.fetch())
assert [i.source_id for i in items] == ["fake:h1"]
assert items[0].raw["search"] == "fake:cph"
assert source.state["search:complete:fake:cph"] == "1"
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin",
"params": {"q": "x", "country": "de"}}]})
assert list(source.fetch()) == []
assert "search:ok:adzuna:berlin" not in source.state
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
sub = store.add_subscription("search", "fake:cph", label="Copenhagen",
board="fake", params={"q": "backend"})
_fake_provider(monkeypatch, ["h1"], complete=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
store.update_subscription(sub.id, subscribed=False, fetch=False)
pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
assert store.roles() == []
# And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]