427 lines
18 KiB
Python
427 lines
18 KiB
Python
"""The `search` plugin over the runner and the provider readers, `country_of`,
|
|
and what the pull does with searches. No network: `_runner.fetch` is a stub,
|
|
and a fake provider stands in where the pull is what is tested. The readers'
|
|
own parsing is tested in `test_readers_<provider>.py`."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import types
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, ClassVar
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from desk.models import Item # noqa: E402
|
|
from app import prep # noqa: E402
|
|
from plugins import readers, search # noqa: E402
|
|
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
|
from plugins.search import SearchSource # noqa: E402
|
|
from app.config import SearchSeed # noqa: E402
|
|
from app.jobs import pull # noqa: E402
|
|
from pipeline import Tally # noqa: E402
|
|
from app.models import country_of, remote_of # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
from prepped import upsert # noqa: E402
|
|
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
# ----------------------------------------------------------------- country
|
|
|
|
@pytest.mark.parametrize(("place", "code"), [
|
|
("Berlin, Germany", "DE"),
|
|
("København K", "DK"),
|
|
("Kongens Lyngby", "DK"),
|
|
("Boston, MA", "US"),
|
|
("Berlin, DE", "DE"), # a state code only when nothing else matched
|
|
("Remote - US", "US"),
|
|
("London | Remote", "GB"),
|
|
("Northern Ireland", "GB"), # the longer name wins over "ireland"
|
|
("Remote - Europe", "EU"),
|
|
("Home based - EMEA", "EU"),
|
|
("London, EMEA", "GB"), # a country beats the region it is in
|
|
("London or New York", ""),
|
|
("Remote", ""),
|
|
("", ""),
|
|
("Wrocław", "PL"), # ł and đ do not decompose under NFKD
|
|
("Łódź, Poland", "PL"),
|
|
("Bogota, CO", "CO"), # a state code that is a country's too
|
|
("Ellwangen, DE", ""), # ... is no US state without a US place
|
|
("Sliema, MT", ""),
|
|
("Casablanca, MA", "MA"),
|
|
("Cambridge, MA", "US"),
|
|
("Chicago, IL", "US"),
|
|
("Madrid, MD, Spain", "ES"),
|
|
("Dublin, London", "EU"), # several European countries
|
|
("Anywhere in France, Belgium, Spain", "EU"),
|
|
("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"),
|
|
("Remote, EU or US", ""), # a region and a country outside it
|
|
("London, Ontario", "CA"), # a province beats the city before it
|
|
("Bellevue, Washington", "US"),
|
|
("Tilburg, NB, Netherlands", "NL"), # a country beats a province code
|
|
("Atlanta, Georgia", "US"),
|
|
("Tbilisi, Georgia", "GE"),
|
|
("Malaysia", "MY"),
|
|
("Belgrade, Serbia", "RS"),
|
|
("Dubai", "AE"),
|
|
("United Arab Emirates", "AE"),
|
|
("Buenos Aires, , Argentina", "AR"),
|
|
("Leicester (GB)", "GB"), # a code in parentheses when nothing else
|
|
("Cagliari (CA)", "IT"), # ... and not over a name
|
|
("Hybrid", ""),
|
|
("Distributed", ""),
|
|
])
|
|
def test_country_of(place: str, code: str) -> None:
|
|
assert country_of(place) == code
|
|
|
|
|
|
@pytest.mark.parametrize(("place", "mode"), [
|
|
("Remote", "remote"),
|
|
("Distributed", "remote"),
|
|
("Home based - Worldwide", "remote"),
|
|
("Remote - US", "remote"),
|
|
("Hybrid", "hybrid"),
|
|
("Remote or hybrid, Berlin", "hybrid"),
|
|
("In-Office", "onsite"),
|
|
("Berlin, Germany", ""),
|
|
("", ""),
|
|
])
|
|
def test_remote_of(place: str, mode: str) -> None:
|
|
assert remote_of(place) == mode
|
|
|
|
|
|
def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None:
|
|
def item(source_id: str, **raw: Any) -> Item:
|
|
it = Item(source_id=source_id, location="https://x.example/1", title="Engineer",
|
|
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw)
|
|
it.source = "ats"
|
|
return it
|
|
|
|
upsert(store, item("a", place="Copenhagen, Denmark"))
|
|
upsert(store, item("b", place="Remote", country="EU"))
|
|
upsert(store, item("c", place="Somewhere"))
|
|
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
|
|
|
|
|
|
# ----------------------------------------------------- read through the runner
|
|
|
|
def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]:
|
|
"""`_runner.fetch` answering `answer(url)`, remembering the URLs asked."""
|
|
asked: list[str] = []
|
|
|
|
def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes:
|
|
assert allow(urlsplit(url).hostname), url
|
|
asked.append(url)
|
|
return answer(url)
|
|
|
|
monkeypatch.setattr(_runner, "fetch", fake_fetch)
|
|
monkeypatch.setattr(search, "PAGE_DELAY", 0)
|
|
return asked
|
|
|
|
|
|
def _jobindex_page(url: str) -> bytes:
|
|
"""A full Jobindex page (20 items) whatever page is asked."""
|
|
page = int(parse_qs(urlsplit(url).query)["page"][0])
|
|
items = "".join(f"<item><title>Role {page}-{n}, Acme</title>"
|
|
f"<link>https://www.jobindex.dk/vis-job/r{page}x{n}</link>"
|
|
f"<description><p>Teaser {n}</p></description></item>"
|
|
for n in range(20))
|
|
return f"<rss><channel><title>Jobindex</title>{items}</channel></rss>".encode()
|
|
|
|
|
|
@pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)])
|
|
def test_the_saved_pages_param_bounds_the_pages_read(
|
|
monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None:
|
|
urls = _web(monkeypatch, _jobindex_page)
|
|
params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else
|
|
{"pages": pages})
|
|
rows, complete = search.run_search("jobindex:cph", params)
|
|
assert len(urls) == asked and len(rows) == 20 * asked
|
|
assert not complete # Jobindex states no total: never complete
|
|
query = parse_qs(urlsplit(urls[0]).query)
|
|
assert (query["q"], query["geoareaid"]) == (["python"], ["1"])
|
|
|
|
|
|
def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes())
|
|
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
|
|
"params": {"q": "x", "pages": 1}}]})
|
|
items = list(source.fetch())
|
|
paid = items[0]
|
|
assert paid.source_id == "jobindex:h1701165"
|
|
assert (paid.body, paid.excerpt) == ("", "")
|
|
assert paid.raw["teaser"].startswith("Do you want to minimise")
|
|
assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK"
|
|
assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}]
|
|
assert paid.raw["place"] == "Hedehusene"
|
|
assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph")
|
|
assert source.state["search:complete:jobindex:cph"] == ""
|
|
|
|
|
|
def _adzuna(count: int | None, stock: int = 60) -> Any:
|
|
def answer(url: str) -> bytes:
|
|
page = int(urlsplit(url).path.rsplit("/", 1)[1])
|
|
start = (page - 1) * 50
|
|
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
|
|
for n in range(start, min(start + 50, stock))]
|
|
body: dict[str, Any] = {"results": results}
|
|
if count is not None:
|
|
body["count"] = count
|
|
return json.dumps(body).encode()
|
|
return answer
|
|
|
|
|
|
def test_a_search_without_its_key_is_refused_before_any_request(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
|
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
|
urls = _web(monkeypatch, _adzuna(60))
|
|
with pytest.raises(ValueError, match="app_id"):
|
|
search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {})
|
|
assert urls == []
|
|
|
|
|
|
def test_the_key_comes_from_the_config_before_the_environment(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("ADZUNA_APP_ID", "env-id")
|
|
monkeypatch.setenv("ADZUNA_APP_KEY", "env-key")
|
|
urls = _web(monkeypatch, _adzuna(60))
|
|
search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
|
|
{"adzuna": {"app_id": "cfg-id"}})
|
|
query = parse_qs(urlsplit(urls[0]).query)
|
|
assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"])
|
|
assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1"
|
|
|
|
|
|
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)])
|
|
def test_an_adzuna_search_is_complete_when_it_read_its_count(
|
|
monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None:
|
|
_web(monkeypatch, _adzuna(count))
|
|
rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
|
|
{"adzuna": {"app_id": "i", "app_key": "k"}})
|
|
assert (len(rows), done) == (60, complete)
|
|
assert rows[0]["place"] == ""
|
|
|
|
|
|
def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def answer(url: str) -> bytes:
|
|
query = parse_qs(urlsplit(url).query)
|
|
start = int(query["offset"][0])
|
|
hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}",
|
|
"workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]}
|
|
for n in range(start, min(start + 100, 200))]
|
|
return json.dumps({"total": {"value": 200}, "hits": hits}).encode()
|
|
|
|
urls = _web(monkeypatch, answer)
|
|
rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"})
|
|
assert (len(rows), done) == (200, True)
|
|
offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls]
|
|
assert offsets == [["0"], ["100"]]
|
|
assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"]
|
|
assert rows[0]["place"] == "Lund; Malmö"
|
|
|
|
|
|
def test_an_unknown_provider_is_refused() -> None:
|
|
with pytest.raises(ValueError, match="provider"):
|
|
search.run_search("indeed:x", {"q": "x"})
|
|
|
|
|
|
def test_provider_of() -> None:
|
|
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
|
|
assert search.provider_of("adzuna:berlin") == "adzuna"
|
|
assert search.provider_of("platsbanken:skane-data") == "platsbanken"
|
|
assert search.provider_of("indeed:x") == ""
|
|
assert search.provider_of("jobindex:") == ""
|
|
|
|
|
|
# ------------------------------------------------------------------- pull
|
|
|
|
class FakeConfig:
|
|
sources: ClassVar[dict[str, object]] = {}
|
|
searches: ClassVar[list[SearchSeed]] = []
|
|
path = None
|
|
|
|
|
|
def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
|
|
"""A provider `fake` that lists `ids` in one page and states a total of
|
|
as many, or of one more when the search is to be incomplete."""
|
|
module: Any = types.ModuleType("fake")
|
|
module.NAME, module.LABEL = "fake", "Fake"
|
|
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
|
|
places="one", fields=Fields(country="stated"), robots="test")
|
|
module.allowed = lambda slug: lambda host: host == "search.fake.example"
|
|
|
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
|
assert ask.params["q"]
|
|
return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None
|
|
|
|
module.page = page
|
|
module.rows = lambda decoded, slug: [
|
|
{"source_id": i, "title": f"Engineer {i}", "company": "Acme",
|
|
"url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}],
|
|
"country": "DK", "remote": None, "posted": None, "description": ""}
|
|
for i in decoded]
|
|
assert not readers.problems(module, "fake")
|
|
monkeypatch.setitem(readers.PROVIDERS, "fake", module)
|
|
|
|
|
|
def _pull(store: Store) -> dict[str, Any]:
|
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
|
closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
|
prep.run(store)
|
|
return closed
|
|
|
|
|
|
def test_a_complete_search_closes_what_it_stopped_listing(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
|
params={"q": "backend"})
|
|
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
|
_pull(store)
|
|
roles = {r.source_id: r for r in store.roles()}
|
|
assert set(roles) == {"fake:h1", "fake:h2"}
|
|
assert roles["fake:h1"].country == "DK"
|
|
assert roles["fake:h1"].subscription_id is not None
|
|
|
|
_fake_provider(monkeypatch, ["h2"], complete=True)
|
|
result = _pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
|
assert (result["searches"], result["closed"]) == (1, 1)
|
|
|
|
|
|
def test_an_incomplete_search_closes_nothing_until_stale(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
|
params={"q": "backend"})
|
|
_fake_provider(monkeypatch, ["h1", "h2"], complete=False)
|
|
_pull(store)
|
|
_fake_provider(monkeypatch, ["h2"], complete=False)
|
|
_pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
|
|
|
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
|
|
_pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
|
|
|
|
|
def test_seed_searches_imports_once(store: Store) -> None:
|
|
config = FakeConfig()
|
|
config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex",
|
|
params={"q": "backend", "area": 1})]
|
|
assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type]
|
|
sub = store.subscriptions(source="search")[0]
|
|
assert (sub.target, sub.params) == ("jobindex:copenhagen-backend",
|
|
{"q": "backend", "area": 1})
|
|
store.unsubscribe(sub.id)
|
|
with store.transaction() as conn:
|
|
conn.execute("DELETE FROM subscriptions")
|
|
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
|
|
|
|
|
|
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_fake_provider(monkeypatch, ["h1"], complete=True)
|
|
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
|
|
"params": {"q": "x"}}]})
|
|
items = list(source.fetch())
|
|
assert [i.source_id for i in items] == ["fake:h1"]
|
|
assert items[0].raw["search"] == "fake:cph"
|
|
assert source.state["search:complete:fake:cph"] == "1"
|
|
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
|
|
|
|
|
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
|
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
|
source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin",
|
|
"params": {"q": "x", "country": "de"}}]})
|
|
assert list(source.fetch()) == []
|
|
assert "search:ok:adzuna:berlin" not in source.state
|
|
|
|
|
|
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
sub = store.add_subscription("search", "fake:cph", label="Copenhagen",
|
|
board="fake", params={"q": "backend"})
|
|
_fake_provider(monkeypatch, ["h1"], complete=False)
|
|
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
|
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
|
store.update_subscription(sub.id, subscribed=False, fetch=False)
|
|
|
|
pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
|
assert store.roles() == []
|
|
# And a batch that starts after the switch reads nothing.
|
|
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
|
|
|
|
|
# ------------------------------------------------- failures and the guard
|
|
|
|
def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The provider `fake` of `_fake_provider`, answering with an error."""
|
|
_fake_provider(monkeypatch, [], complete=True)
|
|
|
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
|
raise ValueError("hits renamed")
|
|
|
|
readers.PROVIDERS["fake"].page = page
|
|
|
|
|
|
def test_a_search_whose_provider_raises_is_recorded_failing(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
|
params={"q": "backend"})
|
|
_fake_provider(monkeypatch, ["h1"], complete=True)
|
|
_pull(store)
|
|
_broken_provider(monkeypatch)
|
|
result = _pull(store)
|
|
assert result["unanswered"] == 1
|
|
since, reason = pull.failing_boards(store)["fake:cph"]
|
|
assert since and reason == "ValueError: hits renamed"
|
|
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
|
|
|
# Failing for `STALE_DAYS` closes its roles, as a board's.
|
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
|
timespec="seconds")
|
|
store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason})
|
|
_pull(store)
|
|
assert store.roles() == []
|
|
|
|
# Its next answer removes the record and reopens what it lists.
|
|
_fake_provider(monkeypatch, ["h1"], complete=True)
|
|
_pull(store)
|
|
assert pull.failing_boards(store) == {}
|
|
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
|
|
|
|
|
def test_a_search_that_lists_nothing_after_forty_is_suspect(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
|
params={"q": "backend"})
|
|
forty = [f"h{n}" for n in range(40)]
|
|
_fake_provider(monkeypatch, forty, complete=True)
|
|
_pull(store)
|
|
_fake_provider(monkeypatch, [], complete=True)
|
|
result = _pull(store)
|
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
|
assert len(store.roles()) == 40
|
|
assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]
|