Runner: readers read from their declaration, ats and search thin over it

plugins/readers/_runner.py takes the guarded fetch, the listing (whole,
paged, one series per filter value unioned by id), completeness from READS,
the sweep of a board over the page cap across pulls, the 10% guard on a
filtered or empty="suspect" read, facets kept a day, the detail reads, the
raw an Item carries and the failure bookkeeping out of plugins/ats.py.
Ask comes from a subscription's params (ask_for, open for the reach), with
a declared key resolved into Ask.params["key"] and refused when missing.

The runtime robots.txt check goes (decided at build time, READS.robots).
plugins/search.py reads the providers of readers.PROVIDERS through the
runner and applies the saved pages param. A filtered board's close keeps
roles with a vote or an application. DETAIL_FIELDS gain places, teaser,
country and employment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:28:33 +02:00
1 parent 779b136757
commit 902089a5e2
14 files changed
+1608 -1492

No files matched your search

+160 -254
View File
@@ -1,15 +1,17 @@
"""The `search` plugin, `country_of`, and what the pull and the CLI do with
searches. No network: Jobindex's feed is a trimmed copy of a real one
(2026-09-23), Adzuna's is written from its documented response, Platsbanken's
is trimmed from a live answer (2026-09-29)."""
"""The `search` plugin over the runner and the provider readers, `country_of`,
and what the pull does with searches. No network: `_runner.fetch` is a stub,
and a fake provider stands in where the pull is what is tested. The readers'
own parsing is tested in `test_readers_<provider>.py`."""
from __future__ import annotations
import json
import sys
import types
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, ClassVar
from urllib.parse import parse_qs, urlsplit
import pytest
@@ -18,7 +20,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from plugins import search # noqa: E402
from plugins import readers, search # noqa: E402
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
@@ -111,246 +114,129 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) ->
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
# ----------------------------------------------------------------- parsers
# ----------------------------------------------------- read through the runner
def test_jobindex_reads_title_company_town_and_date() -> None:
rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes())
assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"]
paid, bare, robot = rows
# A paid ad: the company is its logo's alt text, the title its <h4>.
assert paid["company"] == "DSV"
assert paid["title"].startswith("Dangerous Goods Compliance Manager")
assert paid["place"] == "Hedehusene"
assert paid["country"] == "DK"
assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23"
assert paid["description"].startswith("Do you want to minimise")
# A crawled ad with no company after the title's comma has none.
assert bare["company"] == ""
assert bare["place"] == "København"
# A crawled ad names its company after the title.
assert robot["title"] == "Marketing Content Creator"
assert robot["company"] == "Clerk.IO ApS"
def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]:
"""`_runner.fetch` answering `answer(url)`, remembering the URLs asked."""
asked: list[str] = []
def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes:
assert allow(urlsplit(url).hostname), url
asked.append(url)
return answer(url)
monkeypatch.setattr(_runner, "fetch", fake_fetch)
monkeypatch.setattr(search, "PAGE_DELAY", 0)
return asked
def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None:
assert search._remote("Birkerød, Hybrid position") == "hybrid"
assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
assert search._remote("Remote, Danmark") == "remote"
assert search._remote("Hedehusene") is None
def _jobindex_page(url: str) -> bytes:
"""A full Jobindex page (20 items) whatever page is asked."""
page = int(parse_qs(urlsplit(url).query)["page"][0])
items = "".join(f"<item><title>Role {page}-{n}, Acme</title>"
f"<link>https://www.jobindex.dk/vis-job/r{page}x{n}</link>"
f"<description>&lt;p&gt;Teaser {n}&lt;/p&gt;</description></item>"
for n in range(20))
return f"<rss><channel><title>Jobindex</title>{items}</channel></rss>".encode()
def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None:
rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de")
assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused
first, predicted = rows
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
assert first["description"] == "Build our backend services in Python & Go."
assert first["company"] == "Acme GmbH"
assert first["place"] == "Berlin, Deutschland"
assert first["country"] == "DE"
assert first["remote"] == "remote"
assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR")
assert (predicted["salary_min"], predicted["currency"]) == (None, "")
@pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)])
def test_the_saved_pages_param_bounds_the_pages_read(
monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None:
urls = _web(monkeypatch, _jobindex_page)
params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else
{"pages": pages})
rows, complete = search.run_search("jobindex:cph", params)
assert len(urls) == asked and len(rows) == 20 * asked
assert not complete # Jobindex states no total: never complete
query = parse_qs(urlsplit(urls[0]).query)
assert (query["q"], query["geoareaid"]) == (["python"], ["1"])
def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
with pytest.raises(ValueError, match="app_id"):
search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None:
pages: list[int] = []
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
pages.append(params["page"])
return b"short" if params["page"] == 3 else b"full"
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setattr(search, "_jobindex_page",
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
# Stopped by the page limit with every page full: not complete.
pages.clear()
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert (pages, complete) == ([1, 2], False)
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body(
monkeypatch: pytest.MonkeyPatch) -> None:
"""A full page whose parser dropped one item still has 20 items, so the
search reads on instead of calling itself complete at 19 rows."""
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert not complete
_web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes())
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
"params": {"q": "x", "pages": 1}}]})
items = list(source.fetch())
paid = items[0]
assert paid.source_id == "jobindex:h1701165"
assert (paid.body, paid.excerpt) == ("", "")
assert paid.raw["teaser"].startswith("Do you want to minimise")
assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK"
assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}]
assert paid.raw["place"] == "Hedehusene"
assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph")
assert source.state["search:complete:jobindex:cph"] == ""
def test_a_jobindex_answer_without_a_channel_raises() -> None:
with pytest.raises(ValueError, match="channel"):
search.parse_jobindex(b"<rss version='2.0'></rss>")
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
asked: list[int] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
page = int(url.rsplit("/", 1)[1])
asked.append(page)
start = (page - 1) * search.ADZUNA_PAGE
def _adzuna(count: int | None, stock: int = 60) -> Any:
def answer(url: str) -> bytes:
page = int(urlsplit(url).path.rsplit("/", 1)[1])
start = (page - 1) * 50
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
for n in range(start, min(start + 50, stock))]
body: dict[str, Any] = {"results": results}
if count is not None:
body["count"] = count
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setenv("ADZUNA_APP_ID", "id")
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
return asked
return answer
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
def test_a_search_without_its_key_is_refused_before_any_request(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
urls = _web(monkeypatch, _adzuna(60))
with pytest.raises(ValueError, match="app_id"):
search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {})
assert urls == []
def test_the_key_comes_from_the_config_before_the_environment(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("ADZUNA_APP_ID", "env-id")
monkeypatch.setenv("ADZUNA_APP_KEY", "env-key")
urls = _web(monkeypatch, _adzuna(60))
search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
{"adzuna": {"app_id": "cfg-id"}})
query = parse_qs(urlsplit(urls[0]).query)
assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"])
assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1"
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)])
def test_an_adzuna_search_is_complete_when_it_read_its_count(
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
_adzuna(monkeypatch, count)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None:
_web(monkeypatch, _adzuna(count))
rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
{"adzuna": {"app_id": "i", "app_key": "k"}})
assert (len(rows), done) == (60, complete)
assert rows[0]["place"] == ""
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
#: Two hits of a live `/search?q=data science&region=CaRE_1nn_cSU` (2026-09-29),
#: trimmed to the fields the parser reads, and a third the way an ad with no
#: municipality and a remote headline comes.
PLATSBANKEN = {
"total": {"value": 32},
"hits": [
{"id": "31457942", "headline": "Postdoctoral Researcher",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942",
"publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59",
"description": {"text": "The European Spallation Source (ESS) is one of the largest.",
"text_formatted": "<p>The European Spallation Source (ESS)</p>"},
"employer": {"name": "European Spallation Source Eric"},
"application_details": {"url": "https://europeanspallationsource.se/careers"},
"workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"},
"workplace_address": {"municipality": "Lund", "region": "Skåne län",
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
{"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504",
"publication_date": "2026-09-03T15:46:06",
"description": {"text": "Robotics at ESS."},
"employer": {"name": "European Spallation Source Eric"},
"workplace_model": {"concept_id": None, "label": None},
"workplace_address": {"municipality": None, "region": "Skåne län",
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
{"id": "31400001", "headline": "Data scientist, distansarbete möjligt",
"webpage_url": "javascript:alert(1)", "workplace_address": {}},
{"id": "31400002", "headline": "Data scientist, distansarbete möjligt",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002",
"employer": {"name": None}, "workplace_address": {}},
],
}
def test_platsbanken_reads_title_company_place_and_date() -> None:
rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode())
assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
first, no_town, remote = rows
assert first["title"] == "Postdoctoral Researcher"
assert first["company"] == "European Spallation Source Eric"
assert first["place"] == "Lund"
assert first["country"] == "SE"
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09"
assert first["description"].startswith("The European Spallation Source")
assert first["remote"] is None
assert no_town["place"] == "Skåne län" # the region when there is no municipality
assert remote["remote"] == "remote"
assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None)
def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None,
stock: int = 250) -> list[dict[str, Any]]:
"""Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request,
stating `total`. Returns the params of each request."""
asked: list[dict[str, Any]] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
asked.append(params)
start = params["offset"]
hits = [{"id": str(n), "headline": f"Role {n}",
"webpage_url": f"https://x.example/{n}"}
for n in range(start, min(start + params["limit"], stock))]
body: dict[str, Any] = {"hits": hits}
if total is not None:
body["total"] = {"value": total}
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
return asked
@pytest.mark.parametrize(("total", "pages", "found", "complete"), [
(250, 5, 250, True), # a short third page ends it
(300, 5, 250, False), # the stated total is more than there was
(250, 2, 200, False), # stopped by the page limit
(None, 5, 250, True), # no total: a short page is the end
])
def test_a_platsbanken_search_is_complete_when_it_read_its_total(
monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int,
complete: bool) -> None:
_platsbanken(monkeypatch, total)
rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (found, complete)
def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset(
def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 200, stock=200)
rows, done = search.search_platsbanken(
None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type]
def answer(url: str) -> bytes:
query = parse_qs(urlsplit(url).query)
start = int(query["offset"][0])
hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}",
"workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]}
for n in range(start, min(start + 100, 200))]
return json.dumps({"total": {"value": 200}, "hits": hits}).encode()
urls = _web(monkeypatch, answer)
rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"})
assert (len(rows), done) == (200, True)
assert [p["offset"] for p in asked] == [0, 100]
assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU"
assert asked[0]["limit"] == search.PLATSBANKEN_PAGE
offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls]
assert offsets == [["0"], ["100"]]
assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"]
assert rows[0]["place"] == "Lund; Malmö"
def test_a_platsbanken_search_without_a_region_sends_none(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 5, stock=5)
search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type]
assert "region" not in asked[0]
def test_a_platsbanken_search_never_asks_past_the_offset_limit(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 5000, stock=5000)
monkeypatch.setattr(search, "MAX_PAGES", 30)
_rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type]
assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET
assert not done
def test_a_platsbanken_search_refuses_an_unknown_region() -> None:
with pytest.raises(ValueError, match="region"):
search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type]
def test_platsbanken_regions_are_the_21_lan() -> None:
assert len(search.PLATSBANKEN_REGIONS) == 21
assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län"
def test_an_unknown_provider_is_refused() -> None:
with pytest.raises(ValueError, match="provider"):
search.run_search("indeed:x", {"q": "x"})
def test_provider_of() -> None:
@@ -369,15 +255,27 @@ class FakeConfig:
path = None
def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
def fake(client: object, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København",
"country": "DK", "remote": None, "url": f"https://x.example/{i}",
"posted": None, "description": "", "salary_min": None,
"salary_max": None, "currency": ""} for i in ids], complete
def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
"""A provider `fake` that lists `ids` in one page and states a total of
as many, or of one more when the search is to be incomplete."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL = "fake", "Fake"
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
places="one", fields=Fields(country="stated"), robots="test")
module.allowed = lambda slug: lambda host: host == "search.fake.example"
monkeypatch.setitem(search._SEARCHES, "jobindex", fake)
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
assert ask.params["q"]
return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None
module.page = page
module.rows = lambda decoded, slug: [
{"source_id": i, "title": f"Engineer {i}", "company": "Acme",
"url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}],
"country": "DK", "remote": None, "posted": None, "description": ""}
for i in decoded]
assert not readers.problems(module, "fake")
monkeypatch.setitem(readers.PROVIDERS, "fake", module)
def _pull(store: Store) -> dict[str, Any]:
@@ -388,36 +286,36 @@ def _pull(store: Store) -> dict[str, Any]:
def test_a_complete_search_closes_what_it_stopped_listing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend", "area": 1})
_stub_search(monkeypatch, ["h1", "h2"], complete=True)
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
_pull(store)
roles = {r.source_id: r for r in store.roles()}
assert set(roles) == {"jobindex:h1", "jobindex:h2"}
assert roles["jobindex:h1"].country == "DK"
assert roles["jobindex:h1"].subscription_id is not None
assert set(roles) == {"fake:h1", "fake:h2"}
assert roles["fake:h1"].country == "DK"
assert roles["fake:h1"].subscription_id is not None
_stub_search(monkeypatch, ["h2"], complete=True)
_fake_provider(monkeypatch, ["h2"], complete=True)
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
assert {r.source_id for r in store.roles()} == {"fake:h2"}
assert (result["searches"], result["closed"]) == (1, 1)
def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale(
def test_an_incomplete_search_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_stub_search(monkeypatch, ["h1", "h2"], complete=False)
_fake_provider(monkeypatch, ["h1", "h2"], complete=False)
_pull(store)
_stub_search(monkeypatch, ["h2"], complete=False)
_fake_provider(monkeypatch, ["h2"], complete=False)
_pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"}
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,))
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
_pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
assert {r.source_id for r in store.roles()} == {"fake:h2"}
def test_seed_searches_imports_once(store: Store) -> None:
@@ -434,24 +332,32 @@ def test_seed_searches_imports_once(store: Store) -> None:
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
# -------------------------------------------------------------------- cli
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
_stub_search(monkeypatch, ["h1"], complete=True)
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
_fake_provider(monkeypatch, ["h1"], complete=True)
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
"params": {"q": "x"}}]})
items = list(source.fetch())
assert [i.source_id for i in items] == ["jobindex:h1"]
assert items[0].raw["search"] == "jobindex:cph"
assert source.state["search:complete:jobindex:cph"] == "1"
assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'
assert [i.source_id for i in items] == ["fake:h1"]
assert items[0].raw["search"] == "fake:cph"
assert source.state["search:complete:fake:cph"] == "1"
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin",
"params": {"q": "x", "country": "de"}}]})
assert list(source.fetch()) == []
assert "search:ok:adzuna:berlin" not in source.state
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen",
board="jobindex", params={"q": "backend"})
_stub_search(monkeypatch, ["h1"], complete=False)
sub = store.add_subscription("search", "fake:cph", label="Copenhagen",
board="fake", params={"q": "backend"})
_fake_provider(monkeypatch, ["h1"], complete=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
store.update_subscription(sub.id, subscribed=False, fetch=False)