A second pull (schedule, Run button, `app job pull` in another process) could start while one was fetching and overwrite the stored run marker, so the first run's boards failed their stamp check or a settle closed on the other run's id lists. `claim_run` sets `pull:run` in one conditional statement, only while it equals `pull:settled` or is older than an hour; a refused `pull` job queues no stage and says so in its result and a notice, and the CLI exits 1. Batches carry the marker in their payload, the plugins stamp with it, and the settle reads it off its own batches' payloads before releasing the run. Co-Authored-By: Claude Opus <noreply@anthropic.com>
312 lines
14 KiB
Python
312 lines
14 KiB
Python
"""The `search` plugin, `country_of`, and what the pull and the CLI do with
|
|
searches. No network: Jobindex's feed is a trimmed copy of a real one
|
|
(2026-09-23), Adzuna's is written from its documented response."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, ClassVar
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from desk.models import Item # noqa: E402
|
|
from plugins import search # noqa: E402
|
|
from plugins.search import SearchSource # noqa: E402
|
|
from app.config import SearchSeed # noqa: E402
|
|
from app.jobs import pull # noqa: E402
|
|
from pipeline import Tally # noqa: E402
|
|
from app.models import country_of, remote_of # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
# ----------------------------------------------------------------- country
|
|
|
|
@pytest.mark.parametrize(("place", "code"), [
|
|
("Berlin, Germany", "DE"),
|
|
("København K", "DK"),
|
|
("Kongens Lyngby", "DK"),
|
|
("Boston, MA", "US"),
|
|
("Berlin, DE", "DE"), # a state code only when nothing else matched
|
|
("Remote - US", "US"),
|
|
("London | Remote", "GB"),
|
|
("Northern Ireland", "GB"), # the longer name wins over "ireland"
|
|
("Remote - Europe", "EU"),
|
|
("Home based - EMEA", "EU"),
|
|
("London, EMEA", "GB"), # a country beats the region it is in
|
|
("London or New York", ""),
|
|
("Remote", ""),
|
|
("", ""),
|
|
])
|
|
def test_country_of(place: str, code: str) -> None:
|
|
assert country_of(place) == code
|
|
|
|
|
|
@pytest.mark.parametrize(("place", "mode"), [
|
|
("Remote", "remote"),
|
|
("Distributed", "remote"),
|
|
("Home based - Worldwide", "remote"),
|
|
("Remote - US", "remote"),
|
|
("Hybrid", "hybrid"),
|
|
("Remote or hybrid, Berlin", "hybrid"),
|
|
("In-Office", "onsite"),
|
|
("Berlin, Germany", ""),
|
|
("", ""),
|
|
])
|
|
def test_remote_of(place: str, mode: str) -> None:
|
|
assert remote_of(place) == mode
|
|
|
|
|
|
def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None:
|
|
def item(source_id: str, **raw: Any) -> Item:
|
|
it = Item(source_id=source_id, location="https://x.example/1", title="Engineer",
|
|
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw)
|
|
it.source = "ats"
|
|
return it
|
|
|
|
store.upsert_role(item("a", place="Copenhagen, Denmark"))
|
|
store.upsert_role(item("b", place="Remote", country="EU"))
|
|
store.upsert_role(item("c", place="Somewhere"))
|
|
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
|
|
|
|
|
|
# ----------------------------------------------------------------- parsers
|
|
|
|
def test_jobindex_reads_title_company_town_and_date() -> None:
|
|
rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes())
|
|
assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"]
|
|
paid, bare, robot = rows
|
|
# A paid ad: the company is its logo's alt text, the title its <h4>.
|
|
assert paid["company"] == "DSV"
|
|
assert paid["title"].startswith("Dangerous Goods Compliance Manager")
|
|
assert paid["place"] == "Hedehusene"
|
|
assert paid["country"] == "DK"
|
|
assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23"
|
|
assert paid["description"].startswith("Do you want to minimise")
|
|
# A crawled ad with no company after the title's comma has none.
|
|
assert bare["company"] == ""
|
|
assert bare["place"] == "København"
|
|
# A crawled ad names its company after the title.
|
|
assert robot["title"] == "Marketing Content Creator"
|
|
assert robot["company"] == "Clerk.IO ApS"
|
|
|
|
|
|
def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None:
|
|
assert search._remote("Birkerød, Hybrid position") == "hybrid"
|
|
assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
|
|
assert search._remote("Remote, Danmark") == "remote"
|
|
assert search._remote("Hedehusene") is None
|
|
|
|
|
|
def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None:
|
|
rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de")
|
|
assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused
|
|
first, predicted = rows
|
|
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
|
assert first["description"] == "Build our backend services in Python & Go."
|
|
assert first["company"] == "Acme GmbH"
|
|
assert first["place"] == "Berlin, Deutschland"
|
|
assert first["country"] == "DE"
|
|
assert first["remote"] == "remote"
|
|
assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR")
|
|
assert (predicted["salary_min"], predicted["currency"]) == (None, "")
|
|
|
|
|
|
def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
|
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
|
with pytest.raises(ValueError, match="app_id"):
|
|
search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
|
|
|
|
|
def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
pages: list[int] = []
|
|
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
|
|
|
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
|
pages.append(params["page"])
|
|
return b"short" if params["page"] == 3 else b"full"
|
|
|
|
monkeypatch.setattr(search, "_get", fake_get)
|
|
monkeypatch.setattr(search, "_jobindex_page",
|
|
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
|
|
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
|
|
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
|
|
|
|
# Stopped by the page limit with every page full: not complete.
|
|
pages.clear()
|
|
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
|
assert (pages, complete) == ([1, 2], False)
|
|
|
|
|
|
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A full page whose parser dropped one item still has 20 items, so the
|
|
search reads on instead of calling itself complete at 19 rows."""
|
|
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
|
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
|
|
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
|
|
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
|
assert not complete
|
|
|
|
|
|
def test_a_jobindex_answer_without_a_channel_raises() -> None:
|
|
with pytest.raises(ValueError, match="channel"):
|
|
search.parse_jobindex(b"<rss version='2.0'></rss>")
|
|
|
|
|
|
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
|
|
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
|
|
asked: list[int] = []
|
|
|
|
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
|
page = int(url.rsplit("/", 1)[1])
|
|
asked.append(page)
|
|
start = (page - 1) * search.ADZUNA_PAGE
|
|
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
|
|
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
|
|
body: dict[str, Any] = {"results": results}
|
|
if count is not None:
|
|
body["count"] = count
|
|
return json.dumps(body).encode()
|
|
|
|
monkeypatch.setattr(search, "_get", fake_get)
|
|
monkeypatch.setenv("ADZUNA_APP_ID", "id")
|
|
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
|
|
return asked
|
|
|
|
|
|
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
|
|
def test_an_adzuna_search_is_complete_when_it_read_its_count(
|
|
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
|
|
_adzuna(monkeypatch, count)
|
|
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
|
assert (len(rows), done) == (60, complete)
|
|
|
|
|
|
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
|
|
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
|
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
|
|
|
|
|
|
def test_provider_of() -> None:
|
|
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
|
|
assert search.provider_of("adzuna:berlin") == "adzuna"
|
|
assert search.provider_of("indeed:x") == ""
|
|
assert search.provider_of("jobindex:") == ""
|
|
|
|
|
|
# ------------------------------------------------------------------- pull
|
|
|
|
class FakeConfig:
|
|
sources: ClassVar[dict[str, object]] = {}
|
|
searches: ClassVar[list[SearchSeed]] = []
|
|
path = None
|
|
|
|
|
|
def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
|
|
def fake(client: object, params: dict[str, Any],
|
|
keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
|
|
return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København",
|
|
"country": "DK", "remote": None, "url": f"https://x.example/{i}",
|
|
"posted": None, "description": "", "salary_min": None,
|
|
"salary_max": None, "currency": ""} for i in ids], complete
|
|
|
|
monkeypatch.setitem(search._SEARCHES, "jobindex", fake)
|
|
|
|
|
|
def _pull(store: Store) -> dict[str, Any]:
|
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
|
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
|
|
|
|
|
def test_a_complete_search_closes_what_it_stopped_listing(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
|
|
params={"q": "backend", "area": 1})
|
|
_stub_search(monkeypatch, ["h1", "h2"], complete=True)
|
|
_pull(store)
|
|
roles = {r.source_id: r for r in store.roles()}
|
|
assert set(roles) == {"jobindex:h1", "jobindex:h2"}
|
|
assert roles["jobindex:h1"].country == "DK"
|
|
assert roles["jobindex:h1"].subscription_id is not None
|
|
|
|
_stub_search(monkeypatch, ["h2"], complete=True)
|
|
result = _pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
|
|
assert (result["searches"], result["closed"]) == (1, 1)
|
|
|
|
|
|
def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
|
|
params={"q": "backend"})
|
|
_stub_search(monkeypatch, ["h1", "h2"], complete=False)
|
|
_pull(store)
|
|
_stub_search(monkeypatch, ["h2"], complete=False)
|
|
_pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"}
|
|
|
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,))
|
|
_pull(store)
|
|
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
|
|
|
|
|
|
def test_seed_searches_imports_once(store: Store) -> None:
|
|
config = FakeConfig()
|
|
config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex",
|
|
params={"q": "backend", "area": 1})]
|
|
assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type]
|
|
sub = store.subscriptions(source="search")[0]
|
|
assert (sub.target, sub.params) == ("jobindex:copenhagen-backend",
|
|
{"q": "backend", "area": 1})
|
|
store.unsubscribe(sub.id)
|
|
with store.transaction() as conn:
|
|
conn.execute("DELETE FROM subscriptions")
|
|
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
|
|
|
|
|
|
# -------------------------------------------------------------------- cli
|
|
|
|
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_stub_search(monkeypatch, ["h1"], complete=True)
|
|
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
|
|
"params": {"q": "x"}}]})
|
|
items = list(source.fetch())
|
|
assert [i.source_id for i in items] == ["jobindex:h1"]
|
|
assert items[0].raw["search"] == "jobindex:cph"
|
|
assert source.state["search:complete:jobindex:cph"] == "1"
|
|
assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'
|
|
|
|
|
|
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen",
|
|
board="jobindex", params={"q": "backend"})
|
|
_stub_search(monkeypatch, ["h1"], complete=False)
|
|
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
|
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
|
store.update_subscription(sub.id, subscribed=False, fetch=False)
|
|
|
|
pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
|
assert store.roles() == []
|
|
# And a batch that starts after the switch reads nothing.
|
|
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|