Merge the wiring of the new search providers (wp-src-wiring)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-10-03 12:24:56 +02:00
commit a9c387c130
16 files changed
+422 -39

No files matched your search

+72
View File
@@ -0,0 +1,72 @@
"""Fixtures shared by the app's tests."""
from __future__ import annotations
import sys
from pathlib import Path
from types import ModuleType
from typing import Any
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, ROOT.as_posix())
from plugins import readers # noqa: E402
from plugins.readers import Filters, Reads # noqa: E402
def _stub(name: str, label: str, params: dict[str, str], filters: Filters,
**tables: dict[str, str]) -> ModuleType:
"""A search provider as `plugins/readers/<name>.py` is planned to declare it:
`NAME`, `LABEL`, `PARAMS`, `clean_params` and the tables that fill the
form. Only what `app/` reads, and no listing, since a test that needs one
fakes `plugins.search.run_search`. `clean_params` strips each value, lowers a
country and refuses a region or remote word the tables do not hold."""
module = ModuleType(f"plugins.readers.{name}")
module.NAME, module.LABEL, module.PARAMS = name, label, params
module.READS = Reads(scope="search", listing="paged", page_size=100, ceiling=10000,
text="listing", complete="stated", filters=filters, places="one",
robots="read: stub, checked 2026-10-03")
for table, values in tables.items():
setattr(module, table, values)
allowed = {"region": tables.get("REGIONS"), "remote": tables.get("MODES")
or tables.get("REMOTE")}
def clean_params(given: dict[str, Any]) -> dict[str, Any]:
out = {key: str(value).strip() for key, value in given.items()}
for key, known in allowed.items():
if known is not None and out.get(key) and out[key] not in known:
raise ValueError(f"{key} is not one of {', '.join(known)}")
if "country" in out:
out["country"] = out["country"].lower()
return out
module.clean_params = clean_params
return module
@pytest.fixture
def search_stubs(monkeypatch: pytest.MonkeyPatch) -> dict[str, ModuleType]:
"""freehire, jobnet and jobbank as stub modules in `readers.PROVIDERS`, so
the wiring is tested with or without the real readers merged; a real
module of the name is replaced for the test."""
stubs = {
"freehire": _stub("freehire", "freehire (tech, many countries)",
{"q": "query", "country": "country", "remote": "remote"},
Filters(query="one", country="many", remote="one"),
MODES={"remote": "Remote", "hybrid": "Hybrid", "onsite": "On-site"}),
"jobnet": _stub("jobnet", "Jobnet (Denmark)", {"q": "query", "region": "place"},
Filters(query="one", place="one"),
REGIONS={"HovedstadenOgBornholm": "Hovedstaden og Bornholm",
"Midtjylland": "Midtjylland"}),
"jobbank": _stub("jobbank", "Akademikernes Jobbank (Denmark)",
{"q": "query", "region": "place", "remote": "remote"},
Filters(query="one", place="one", remote="one"),
REGIONS={"2": "Storkøbenhavn", "8": "Østjylland"},
REMOTE={"helt": "Fully remote", "delvist": "Partly remote"}),
}
for name, module in stubs.items():
monkeypatch.setitem(readers.PROVIDERS, name, module)
return stubs
+12
View File
@@ -181,6 +181,18 @@ def test_keep_order_ranks():
key=keep_order).uid == "a"
def test_every_search_provider_loses_to_a_board(search_stubs):
from app.dupes import is_search_host
assert all(is_search_host(h) for h in
("jobindex", "adzuna", "platsbanken", "freehire", "jobnet", "jobbank"))
assert not any(is_search_host(h) for h in ("greenhouse", "workday", "teamtailor", ""))
board = cp("z", host="greenhouse", desc=False, posted="2026-09-09")
for host in ("freehire", "jobnet", "jobbank"):
search = cp("a", host=host, desc=True, posted="2026-01-01")
assert min([search, board], key=keep_order) is board
def test_a_row_without_a_town_joins_only_one_group():
# "Germany" matches Berlin and Munich; joining both would make them one role.
rows = [cp("a", place="Berlin"), cp("b", place="Munich"), cp("c", place="Germany")]
+15
View File
@@ -61,6 +61,21 @@ def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store)
assert dup_of(store) == {board.uid: None, search.uid: board.uid}
def test_a_freehire_copy_of_a_greenhouse_role_is_hidden_behind_the_board(
store: Store, search_stubs: dict[str, object]) -> None:
"""freehire aggregates other hosts' postings; the same role it lists from
a board roledesk reads is one role, and the board's copy is the one kept
even though freehire's is the one with a description."""
board = role("board", company="Lunar", title="Head of Data", place="Copenhagen",
board="greenhouse", body="", day=22)
copy = role("copy", company="Lunar", title="Head of Data", place="Copenhagen, Denmark",
board=None, provider="freehire", source="greenhouse", day=20)
for item in (copy, board):
store.upsert_role(item)
store.rebuild_dupes()
assert dup_of(store) == {board.uid: None, copy.uid: board.uid}
def kept_of(store: Store, *items: Item) -> str:
"""The uid of the one of these roles that is not a copy."""
found = dup_of(store)
+71 -1
View File
@@ -576,7 +576,7 @@ def test_search_params_keeps_only_what_the_provider_reads() -> None:
def test_the_providers_and_their_params_follow_the_reader_declarations() -> None:
from plugins import readers
assert sources.PROVIDERS == tuple(readers.PROVIDERS) == ("jobindex", "adzuna", "platsbanken")
assert tuple(readers.PROVIDERS)[:3] == ("jobindex", "adzuna", "platsbanken")
assert sources.search_params("adzuna", " backend ", country=" DE ", where=" Berlin ",
area=1, region="x", pages=2) == {
"q": "backend", "country": "de", "where": "Berlin", "pages": 2}
@@ -589,3 +589,73 @@ def test_search_params_refuses_a_region_that_is_no_swedish_lan() -> None:
sources.search_params("platsbanken", "data", region="zYpL_aGk_QY8") # Midland
with pytest.raises(ValueError, match="provider"):
sources.search_params("indeed", "data")
def test_search_params_of_the_three_newer_providers_follow_their_declarations(
search_stubs: dict[str, Any]) -> None:
assert sources.search_params("freehire", " head of data ", country=" DK,se ",
remote="remote", area=1, region="x", pages=2) == {
"q": "head of data", "country": "dk,se", "remote": "remote", "pages": 2}
assert sources.search_params("jobnet", "data", region="Midtjylland", country="dk",
remote="remote") == {"q": "data", "region": "Midtjylland"}
assert sources.search_params("jobbank", "data", region="2", remote="helt") == {
"q": "data", "region": "2", "remote": "helt"}
with pytest.raises(ValueError, match="region"):
sources.search_params("jobnet", "data", region="Fyn")
with pytest.raises(ValueError, match="remote"):
sources.search_params("freehire", "data", remote="helt")
def test_the_provider_error_lists_the_loaded_readers(search_stubs: dict[str, Any]) -> None:
with pytest.raises(ValueError, match=r"freehire.*jobnet.*jobbank"):
sources.search_params("indeed", "data")
def test_a_provider_table_is_empty_for_a_reader_that_is_not_loaded(
search_stubs: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None:
from plugins import readers
assert sources.provider_table("jobbank", "REMOTE") == {
"helt": "Fully remote", "delvist": "Partly remote"}
assert sources.provider_table("freehire", "REGIONS") == {} # it declares none
monkeypatch.delitem(readers.PROVIDERS, "jobbank")
assert sources.provider_table("jobbank", "REGIONS") == {}
assert sources.provider_table("nobody", "MODES") == {}
def test_cli_searches_add_takes_the_newer_providers_and_their_flags(
search_stubs: dict[str, Any], store: Store, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sources, "try_search", lambda config, provider, name, params: (
[{"title": "Data lead", "company": "Acme", "place": "København",
"url": "https://x.example/1"}], True))
config = tmp_path / "config.yaml"
config.write_text(f"database: {store.path}\n")
def add(*argv: str) -> int:
return main(["-c", str(config), "searches", "add", *argv])
assert add("freehire", "Head of data", "--q", "head of data", "--country", "DK,SE",
"--remote", "hybrid") == 0
assert add("jobnet", "Data science", "--q", "data science", "--region",
"HovedstadenOgBornholm") == 0
assert add("jobbank", "Data director", "--q", "data director", "--region", "2",
"--remote", "delvist") == 0
capsys.readouterr()
saved = {s.board: s.params for s in store.subscriptions(source="search")}
assert saved == {
"freehire": {"q": "head of data", "country": "dk,se", "remote": "hybrid"},
"jobnet": {"q": "data science", "region": "HovedstadenOgBornholm"},
"jobbank": {"q": "data director", "region": "2", "remote": "delvist"}}
assert add("jobnet", "Bad", "--q", "x", "--region", "Fyn") == 1
assert "region" in capsys.readouterr().err
def test_cli_searches_add_help_names_the_regions_and_work_modes(
search_stubs: dict[str, Any], capsys: pytest.CaptureFixture[str]) -> None:
with pytest.raises(SystemExit):
main(["searches", "add", "-h"])
text = " ".join(capsys.readouterr().out.split())
assert "freehire" in text and "jobnet" in text and "jobbank" in text
assert "HovedstadenOgBornholm" in text and "2 Storkøbenhavn" in text
assert "hybrid Hybrid" in text and "helt Fully remote" in text
+51
View File
@@ -1357,6 +1357,57 @@ def test_a_search_is_tried_saved_updated_and_removed(
assert again["source"]["id"] == sub_id and again["source"]["on"] is True
def test_the_newer_providers_save_with_their_own_params(
client: TestClient, store: Store, search_stubs: dict[str, Any],
monkeypatch: pytest.MonkeyPatch) -> None:
_fake_search(monkeypatch, 2)
freehire = client.post("/api/searches", json={
"provider": "freehire", "name": "Head of data", "q": "head of data",
"country": "DK,se", "remote": "hybrid", "area": 1, "region": "x"}).json()
assert freehire["source"]["target"] == "freehire:head-of-data"
assert freehire["source"]["params"] == {"q": "head of data", "country": "dk,se",
"remote": "hybrid"}
jobnet = client.post("/api/searches/try", json={
"provider": "jobnet", "q": "data", "region": "Midtjylland", "remote": "helt"})
assert jobnet.status_code == 200 and jobnet.json()["found"] == 2
jobbank = client.post("/api/searches", json={
"provider": "jobbank", "name": "Data director", "q": "data director", "region": "2",
"remote": "helt"}).json()
assert jobbank["source"]["params"] == {"q": "data director", "region": "2", "remote": "helt"}
refused = client.post("/api/searches", json={
"provider": "jobnet", "name": "Bad", "q": "data", "region": "Fyn"})
assert refused.status_code == 422 and "region" in refused.json()["detail"]
assert [s.board for s in store.subscriptions(source="search")] == ["freehire", "jobbank"]
def test_the_sources_form_offers_the_providers_that_are_loaded(
client: TestClient, search_stubs: dict[str, Any]) -> None:
page = client.get("/settings/sources").text
for option in ('<option value="freehire">freehire (tech, many countries)</option>',
'<option value="jobnet">Jobnet (Denmark)</option>',
'<option value="jobbank">Akademikernes Jobbank (Denmark)</option>',
'<option value="HovedstadenOgBornholm">Hovedstaden og Bornholm</option>',
'<option value="2">Storkøbenhavn</option>',
'<option value="hybrid">Hybrid</option>',
'<option value="delvist">Partly remote</option>'):
assert option in page
for control in ('id="s-countries"', 'id="s-mode"', 'id="s-region-jobnet"',
'id="s-region-jobbank"', 'id="s-remote-jobbank"', 'id="s-region"'):
assert page.count(control) == 1
assert 'value="jobindex"' in page and 'value="platsbanken"' in page
def test_the_sources_form_renders_without_the_newer_providers(
client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
from plugins import readers
for name in ("freehire", "jobnet", "jobbank"):
monkeypatch.delitem(readers.PROVIDERS, name, raising=False)
page = client.get("/settings/sources").text
assert 'value="adzuna"' in page and 'value="freehire"' not in page
assert 'id="s-region-jobnet"' in page and 'value="HovedstadenOgBornholm"' not in page
def test_a_search_that_finds_nothing_or_is_malformed_is_refused(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_fake_search(monkeypatch, 0)