Files
jobboard/tests/test_search.py
T
WolfandClaude Fable 5.1 fb83ae5099 Gates reject on evidence only, any gates are one group, remote is read off the location line
A require gate on an empty column gated the role ("country unknown"), and extract, the stage that fills the column, is bounded to the roles that rank, so 2,281 of 15,556 open roles could never come back: every board that writes "Remote", "Hybrid" or "Distributed" as its location. An empty column now neither passes nor fails a gate; the role ranks, extract reads it, the rescore decides.

`any` is a third gate mode: the any gates are one group a role must match one of, so a ranking can say "in Denmark, or remote", which require alone could not. The group is gated only when some member had evidence against it.

`remote_of(place)` fills `remote` from the location line the way `country_of` fills `country`, and the startup backfill does both, since Greenhouse writes "Hybrid" as the location and gives no flag.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-28 19:30:06 +02:00

266 lines
11 KiB
Python

"""The `search` plugin, `country_of`, and what the pull and the CLI do with
searches. No network: Jobindex's feed is a trimmed copy of a real one
(2026-09-23), Adzuna's is written from its documented response."""
from __future__ import annotations
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, ClassVar
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from plugins import search # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
# ----------------------------------------------------------------- country
@pytest.mark.parametrize(("place", "code"), [
("Berlin, Germany", "DE"),
("København K", "DK"),
("Kongens Lyngby", "DK"),
("Boston, MA", "US"),
("Berlin, DE", "DE"), # a state code only when nothing else matched
("Remote - US", "US"),
("London | Remote", "GB"),
("Northern Ireland", "GB"), # the longer name wins over "ireland"
("Remote - Europe", "EU"),
("Home based - EMEA", "EU"),
("London, EMEA", "GB"), # a country beats the region it is in
("London or New York", ""),
("Remote", ""),
("", ""),
])
def test_country_of(place: str, code: str) -> None:
assert country_of(place) == code
@pytest.mark.parametrize(("place", "mode"), [
("Remote", "remote"),
("Distributed", "remote"),
("Home based - Worldwide", "remote"),
("Remote - US", "remote"),
("Hybrid", "hybrid"),
("Remote or hybrid, Berlin", "hybrid"),
("In-Office", "onsite"),
("Berlin, Germany", ""),
("", ""),
])
def test_remote_of(place: str, mode: str) -> None:
assert remote_of(place) == mode
def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None:
def item(source_id: str, **raw: Any) -> Item:
it = Item(source_id=source_id, location="https://x.example/1", title="Engineer",
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw)
it.source = "ats"
return it
store.upsert_role(item("a", place="Copenhagen, Denmark"))
store.upsert_role(item("b", place="Remote", country="EU"))
store.upsert_role(item("c", place="Somewhere"))
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
# ----------------------------------------------------------------- parsers
def test_jobindex_reads_title_company_town_and_date() -> None:
rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes())
assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"]
paid, bare, robot = rows
# A paid ad: the company is its logo's alt text, the title its <h4>.
assert paid["company"] == "DSV"
assert paid["title"].startswith("Dangerous Goods Compliance Manager")
assert paid["place"] == "Hedehusene"
assert paid["country"] == "DK"
assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23"
assert paid["description"].startswith("Do you want to minimise")
# A crawled ad with no company after the title's comma has none.
assert bare["company"] == ""
assert bare["place"] == "København"
# A crawled ad names its company after the title.
assert robot["title"] == "Marketing Content Creator"
assert robot["company"] == "Clerk.IO ApS"
def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None:
assert search._remote("Birkerød, Hybrid position") == "hybrid"
assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
assert search._remote("Remote, Danmark") == "remote"
assert search._remote("Hedehusene") is None
def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None:
rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de")
assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused
first, predicted = rows
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
assert first["description"] == "Build our backend services in Python & Go."
assert first["company"] == "Acme GmbH"
assert first["place"] == "Berlin, Deutschland"
assert first["country"] == "DE"
assert first["remote"] == "remote"
assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR")
assert (predicted["salary_min"], predicted["currency"]) == (None, "")
def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
with pytest.raises(ValueError, match="app_id"):
search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None:
pages: list[int] = []
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
pages.append(params["page"])
return b"short" if params["page"] == 3 else b"full"
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setattr(search, "parse_jobindex",
lambda body: full[:5] if body == b"short" else full)
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
pages.clear()
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert (pages, complete) == ([1, 2], False)
def test_provider_of() -> None:
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
assert search.provider_of("adzuna:berlin") == "adzuna"
assert search.provider_of("indeed:x") == ""
assert search.provider_of("jobindex:") == ""
# ------------------------------------------------------------------- pull
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[SearchSeed]] = []
path = None
def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
def fake(client: object, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København",
"country": "DK", "remote": None, "url": f"https://x.example/{i}",
"posted": None, "description": "", "salary_min": None,
"salary_max": None, "currency": ""} for i in ids], complete
monkeypatch.setitem(search._SEARCHES, "jobindex", fake)
def _pull(store: Store) -> dict[str, Any]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
for batch in batches:
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
def test_a_complete_search_closes_what_it_stopped_listing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend", "area": 1})
_stub_search(monkeypatch, ["h1", "h2"], complete=True)
_pull(store)
roles = {r.source_id: r for r in store.roles()}
assert set(roles) == {"jobindex:h1", "jobindex:h2"}
assert roles["jobindex:h1"].country == "DK"
assert roles["jobindex:h1"].subscription_id is not None
_stub_search(monkeypatch, ["h2"], complete=True)
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
assert (result["searches"], result["closed"]) == (1, 1)
def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend"})
_stub_search(monkeypatch, ["h1", "h2"], complete=False)
_pull(store)
_stub_search(monkeypatch, ["h2"], complete=False)
_pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"}
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,))
_pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
def test_seed_searches_imports_once(store: Store) -> None:
config = FakeConfig()
config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex",
params={"q": "backend", "area": 1})]
assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type]
sub = store.subscriptions(source="search")[0]
assert (sub.target, sub.params) == ("jobindex:copenhagen-backend",
{"q": "backend", "area": 1})
store.unsubscribe(sub.id)
with store.transaction() as conn:
conn.execute("DELETE FROM subscriptions")
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
# -------------------------------------------------------------------- cli
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
_stub_search(monkeypatch, ["h1"], complete=True)
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
"params": {"q": "x"}}]})
items = list(source.fetch())
assert [i.source_id for i in items] == ["jobindex:h1"]
assert items[0].raw["search"] == "jobindex:cph"
assert source.state["search:complete:jobindex:cph"] == "1"
assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen",
board="jobindex", params={"q": "backend"})
_stub_search(monkeypatch, ["h1"], complete=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
store.update_subscription(sub.id, subscribed=False, fetch=False)
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
assert store.roles() == []
# And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type]