About 190 German towns (mostly 1KOMMA5°'s) and the German states, French, Italian, Dutch, Asian and other towns, the ISO3 codes and German or French country names boards write. Towns keep their marks through spelling groups. The tests that used Paderborn, Jena and Überlingen as unknown towns use Bopfingen, Lauchheim and Ellwangen instead. Left: names that are no place, ambiguous ones (Halifax, Richmond, Victoria, Durham, Warwick) and bare codes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
374 lines
16 KiB
Python
374 lines
16 KiB
Python
"""The `search` plugin over the runner and the provider readers, `country_of`,
|
|
and what the pull does with searches. No network: `_runner.fetch` is a stub,
|
|
and a fake provider stands in where the pull is what is tested. The readers'
|
|
own parsing is tested in `test_readers_<provider>.py`."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import types
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, ClassVar
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from desk.models import Item # noqa: E402
|
|
from app import prep # noqa: E402
|
|
from plugins import readers, search # noqa: E402
|
|
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
|
from plugins.search import SearchSource # noqa: E402
|
|
from app.config import SearchSeed # noqa: E402
|
|
from app.jobs import pull # noqa: E402
|
|
from pipeline import Tally # noqa: E402
|
|
from app.models import country_of, remote_of # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
from prepped import upsert # noqa: E402
|
|
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
# ----------------------------------------------------------------- country
|
|
|
|
@pytest.mark.parametrize(("place", "code"), [
|
|
("Berlin, Germany", "DE"),
|
|
("København K", "DK"),
|
|
("Kongens Lyngby", "DK"),
|
|
("Boston, MA", "US"),
|
|
("Berlin, DE", "DE"), # a state code only when nothing else matched
|
|
("Remote - US", "US"),
|
|
("London | Remote", "GB"),
|
|
("Northern Ireland", "GB"), # the longer name wins over "ireland"
|
|
("Remote - Europe", "EU"),
|
|
("Home based - EMEA", "EU"),
|
|
("London, EMEA", "GB"), # a country beats the region it is in
|
|
("London or New York", ""),
|
|
("Remote", ""),
|
|
("", ""),
|
|
("Wrocław", "PL"), # ł and đ do not decompose under NFKD
|
|
("Łódź, Poland", "PL"),
|
|
("Bogota, CO", "CO"), # a state code that is a country's too
|
|
("Ellwangen, DE", ""), # ... is no US state without a US place
|
|
("Sliema, MT", ""),
|
|
("Casablanca, MA", "MA"),
|
|
("Cambridge, MA", "US"),
|
|
("Chicago, IL", "US"),
|
|
("Madrid, MD, Spain", "ES"),
|
|
("Dublin, London", "EU"), # several European countries
|
|
("Anywhere in France, Belgium, Spain", "EU"),
|
|
("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"),
|
|
("Remote, EU or US", ""), # a region and a country outside it
|
|
("London, Ontario", "CA"), # a province beats the city before it
|
|
("Bellevue, Washington", "US"),
|
|
("Tilburg, NB, Netherlands", "NL"), # a country beats a province code
|
|
("Atlanta, Georgia", "US"),
|
|
("Tbilisi, Georgia", "GE"),
|
|
("Malaysia", "MY"),
|
|
("Belgrade, Serbia", "RS"),
|
|
("Dubai", "AE"),
|
|
("United Arab Emirates", "AE"),
|
|
("Buenos Aires, , Argentina", "AR"),
|
|
("Leicester (GB)", "GB"), # a code in parentheses when nothing else
|
|
("Cagliari (CA)", "IT"), # ... and not over a name
|
|
("Hybrid", ""),
|
|
("Distributed", ""),
|
|
])
|
|
def test_country_of(place: str, code: str) -> None:
|
|
assert country_of(place) == code
|
|
|
|
|
|
@pytest.mark.parametrize(("place", "mode"), [
|
|
("Remote", "remote"),
|
|
("Distributed", "remote"),
|
|
("Home based - Worldwide", "remote"),
|
|
("Remote - US", "remote"),
|
|
("Hybrid", "hybrid"),
|
|
("Remote or hybrid, Berlin", "hybrid"),
|
|
("In-Office", "onsite"),
|
|
("Berlin, Germany", ""),
|
|
("", ""),
|
|
])
|
|
def test_remote_of(place: str, mode: str) -> None:
|
|
assert remote_of(place) == mode
|
|
|
|
|
|
def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None:
|
|
def item(source_id: str, **raw: Any) -> Item:
|
|
it = Item(source_id=source_id, location="https://x.example/1", title="Engineer",
|
|
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw)
|
|
it.source = "ats"
|
|
return it
|
|
|
|
upsert(store, item("a", place="Copenhagen, Denmark"))
|
|
upsert(store, item("b", place="Remote", country="EU"))
|
|
upsert(store, item("c", place="Somewhere"))
|
|
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
|
|
|
|
|
|
# ----------------------------------------------------- read through the runner
|
|
|
|
def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]:
|
|
"""`_runner.fetch` answering `answer(url)`, remembering the URLs asked."""
|
|
asked: list[str] = []
|
|
|
|
def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes:
|
|
assert allow(urlsplit(url).hostname), url
|
|
asked.append(url)
|
|
return answer(url)
|
|
|
|
monkeypatch.setattr(_runner, "fetch", fake_fetch)
|
|
monkeypatch.setattr(search, "PAGE_DELAY", 0)
|
|
return asked
|
|
|
|
|
|
def _jobindex_page(url: str) -> bytes:
|
|
"""A full Jobindex page (20 items) whatever page is asked."""
|
|
page = int(parse_qs(urlsplit(url).query)["page"][0])
|
|
items = "".join(f"<item><title>Role {page}-{n}, Acme</title>"
|
|
f"<link>https://www.jobindex.dk/vis-job/r{page}x{n}</link>"
|
|
f"<description><p>Teaser {n}</p></description></item>"
|
|
for n in range(20))
|
|
return f"<rss><channel><title>Jobindex</title>{items}</channel></rss>".encode()
|
|
|
|
|
|
@pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)])
|
|
def test_the_saved_pages_param_bounds_the_pages_read(
|
|
monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None:
|
|
urls = _web(monkeypatch, _jobindex_page)
|
|
params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else
|
|
{"pages": pages})
|
|
rows, complete = search.run_search("jobindex:cph", params)
|
|
assert len(urls) == asked and len(rows) == 20 * asked
|
|
assert not complete # Jobindex states no total: never complete
|
|
query = parse_qs(urlsplit(urls[0]).query)
|
|
assert (query["q"], query["geoareaid"]) == (["python"], ["1"])
|
|
|
|
|
|
def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes())
|
|
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
|
|
"params": {"q": "x", "pages": 1}}]})
|
|
items = list(source.fetch())
|
|
paid = items[0]
|
|
assert paid.source_id == "jobindex:h1701165"
|
|
assert (paid.body, paid.excerpt) == ("", "")
|
|
assert paid.raw["teaser"].startswith("Do you want to minimise")
|
|
assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK"
|
|
assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}]
|
|
assert paid.raw["place"] == "Hedehusene"
|
|
assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph")
|
|
assert source.state["search:complete:jobindex:cph"] == ""
|
|
|
|
|
|
def _adzuna(count: int | None, stock: int = 60) -> Any:
|
|
def answer(url: str) -> bytes:
|
|
page = int(urlsplit(url).path.rsplit("/", 1)[1])
|
|
start = (page - 1) * 50
|
|
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
|
|
for n in range(start, min(start + 50, stock))]
|
|
body: dict[str, Any] = {"results": results}
|
|
if count is not None:
|
|
body["count"] = count
|
|
return json.dumps(body).encode()
|
|
return answer
|
|
|
|
|
|
def test_a_search_without_its_key_is_refused_before_any_request(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
|
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
|
urls = _web(monkeypatch, _adzuna(60))
|
|
with pytest.raises(ValueError, match="app_id"):
|
|
search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {})
|
|
assert urls == []
|
|
|
|
|
|
def test_the_key_comes_from_the_config_before_the_environment(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("ADZUNA_APP_ID", "env-id")
|
|
monkeypatch.setenv("ADZUNA_APP_KEY", "env-key")
|
|
urls = _web(monkeypatch, _adzuna(60))
|
|
search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
|
|
{"adzuna": {"app_id": "cfg-id"}})
|
|
query = parse_qs(urlsplit(urls[0]).query)
|
|
assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"])
|
|
assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1"
|
|
|
|
|
|
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)])
|
|
def test_an_adzuna_search_is_complete_when_it_read_its_count(
|
|
monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None:
|
|
_web(monkeypatch, _adzuna(count))
|
|
rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
|
|
{"adzuna": {"app_id": "i", "app_key": "k"}})
|
|
assert (len(rows), done) == (60, complete)
|
|
assert rows[0]["place"] == ""
|
|
|
|
|
|
def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def answer(url: str) -> bytes:
|
|
query = parse_qs(urlsplit(url).query)
|
|
start = int(query["offset"][0])
|
|
hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}",
|
|
"workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]}
|
|
for n in range(start, min(start + 100, 200))]
|
|
return json.dumps({"total": {"value": 200}, "hits": hits}).encode()
|
|
|
|
urls = _web(monkeypatch, answer)
|
|
rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"})
|
|
assert (len(rows), done) == (200, True)
|
|
offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls]
|
|
assert offsets == [["0"], ["100"]]
|
|
assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"]
|
|
assert rows[0]["place"] == "Lund; Malmö"
|
|
|
|
|
|
def test_an_unknown_provider_is_refused() -> None:
|
|
with pytest.raises(ValueError, match="provider"):
|
|
search.run_search("indeed:x", {"q": "x"})
|
|
|
|
|
|
def test_provider_of() -> None:
|
|
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
|
|
assert search.provider_of("adzuna:berlin") == "adzuna"
|
|
assert search.provider_of("platsbanken:skane-data") == "platsbanken"
|
|
assert search.provider_of("indeed:x") == ""
|
|
assert search.provider_of("jobindex:") == ""
|
|
|
|
|
|
# ------------------------------------------------------------------- pull
|
|
|
|
class FakeConfig:
|
|
sources: ClassVar[dict[str, object]] = {}
|
|
searches: ClassVar[list[SearchSeed]] = []
|
|
path = None
|
|
|
|
|
|
def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
|
|
"""A provider `fake` that lists `ids` in one page and states a total of
|
|
as many, or of one more when the search is to be incomplete."""
|
|
module: Any = types.ModuleType("fake")
|
|
module.NAME, module.LABEL = "fake", "Fake"
|
|
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
|
|
places="one", fields=Fields(country="stated"), robots="test")
|
|
module.allowed = lambda slug: lambda host: host == "search.fake.example"
|
|
|
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
|
assert ask.params["q"]
|
|
return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None
|
|
|
|
module.page = page
|
|
module.rows = lambda decoded, slug: [
|
|
{"source_id": i, "title": f"Engineer {i}", "company": "Acme",
|
|
"url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}],
|
|
"country": "DK", "remote": None, "posted": None, "description": ""}
|
|
for i in decoded]
|
|
assert not readers.problems(module, "fake")
|
|
monkeypatch.setitem(readers.PROVIDERS, "fake", module)
|
|
|
|
|
|
def _pull(store: Store) -> dict[str, Any]:
|
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
|
closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
|
prep.run(store)
|
|
return closed
|
|
|
|
|
|
def test_a_complete_search_closes_what_it_stopped_listing(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
|
params={"q": "backend"})
|
|
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
|
_pull(store)
|
|
roles = {r.source_id: r for r in store.roles()}
|
|
assert set(roles) == {"fake:h1", "fake:h2"}
|
|
assert roles["fake:h1"].country == "DK"
|
|
assert roles["fake:h1"].subscription_id is not None
|
|
|
|
_fake_provider(monkeypatch, ["h2"], complete=True)
|
|
result = _pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
|
assert (result["searches"], result["closed"]) == (1, 1)
|
|
|
|
|
|
def test_an_incomplete_search_closes_nothing_until_stale(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
|
params={"q": "backend"})
|
|
_fake_provider(monkeypatch, ["h1", "h2"], complete=False)
|
|
_pull(store)
|
|
_fake_provider(monkeypatch, ["h2"], complete=False)
|
|
_pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
|
|
|
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
|
|
_pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
|
|
|
|
|
def test_seed_searches_imports_once(store: Store) -> None:
|
|
config = FakeConfig()
|
|
config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex",
|
|
params={"q": "backend", "area": 1})]
|
|
assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type]
|
|
sub = store.subscriptions(source="search")[0]
|
|
assert (sub.target, sub.params) == ("jobindex:copenhagen-backend",
|
|
{"q": "backend", "area": 1})
|
|
store.unsubscribe(sub.id)
|
|
with store.transaction() as conn:
|
|
conn.execute("DELETE FROM subscriptions")
|
|
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
|
|
|
|
|
|
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_fake_provider(monkeypatch, ["h1"], complete=True)
|
|
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
|
|
"params": {"q": "x"}}]})
|
|
items = list(source.fetch())
|
|
assert [i.source_id for i in items] == ["fake:h1"]
|
|
assert items[0].raw["search"] == "fake:cph"
|
|
assert source.state["search:complete:fake:cph"] == "1"
|
|
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
|
|
|
|
|
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
|
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
|
source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin",
|
|
"params": {"q": "x", "country": "de"}}]})
|
|
assert list(source.fetch()) == []
|
|
assert "search:ok:adzuna:berlin" not in source.state
|
|
|
|
|
|
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
sub = store.add_subscription("search", "fake:cph", label="Copenhagen",
|
|
board="fake", params={"q": "backend"})
|
|
_fake_provider(monkeypatch, ["h1"], complete=False)
|
|
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
|
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
|
store.update_subscription(sub.id, subscribed=False, fetch=False)
|
|
|
|
pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
|
assert store.roles() == []
|
|
# And a batch that starts after the switch reads nothing.
|
|
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|