Merge the wiring of the new search providers (wp-src-wiring)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
a9c387c130
16 files changed
+422
-39
No files matched your search
@@ -0,0 +1,72 @@
|
||||
"""Fixtures shared by the app's tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, ROOT.as_posix())
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Filters, Reads # noqa: E402
|
||||
|
||||
|
||||
def _stub(name: str, label: str, params: dict[str, str], filters: Filters,
|
||||
**tables: dict[str, str]) -> ModuleType:
|
||||
"""A search provider as `plugins/readers/<name>.py` is planned to declare it:
|
||||
`NAME`, `LABEL`, `PARAMS`, `clean_params` and the tables that fill the
|
||||
form. Only what `app/` reads, and no listing, since a test that needs one
|
||||
fakes `plugins.search.run_search`. `clean_params` strips each value, lowers a
|
||||
country and refuses a region or remote word the tables do not hold."""
|
||||
module = ModuleType(f"plugins.readers.{name}")
|
||||
module.NAME, module.LABEL, module.PARAMS = name, label, params
|
||||
module.READS = Reads(scope="search", listing="paged", page_size=100, ceiling=10000,
|
||||
text="listing", complete="stated", filters=filters, places="one",
|
||||
robots="read: stub, checked 2026-10-03")
|
||||
for table, values in tables.items():
|
||||
setattr(module, table, values)
|
||||
allowed = {"region": tables.get("REGIONS"), "remote": tables.get("MODES")
|
||||
or tables.get("REMOTE")}
|
||||
|
||||
def clean_params(given: dict[str, Any]) -> dict[str, Any]:
|
||||
out = {key: str(value).strip() for key, value in given.items()}
|
||||
for key, known in allowed.items():
|
||||
if known is not None and out.get(key) and out[key] not in known:
|
||||
raise ValueError(f"{key} is not one of {', '.join(known)}")
|
||||
if "country" in out:
|
||||
out["country"] = out["country"].lower()
|
||||
return out
|
||||
|
||||
module.clean_params = clean_params
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def search_stubs(monkeypatch: pytest.MonkeyPatch) -> dict[str, ModuleType]:
|
||||
"""freehire, jobnet and jobbank as stub modules in `readers.PROVIDERS`, so
|
||||
the wiring is tested with or without the real readers merged; a real
|
||||
module of the name is replaced for the test."""
|
||||
stubs = {
|
||||
"freehire": _stub("freehire", "freehire (tech, many countries)",
|
||||
{"q": "query", "country": "country", "remote": "remote"},
|
||||
Filters(query="one", country="many", remote="one"),
|
||||
MODES={"remote": "Remote", "hybrid": "Hybrid", "onsite": "On-site"}),
|
||||
"jobnet": _stub("jobnet", "Jobnet (Denmark)", {"q": "query", "region": "place"},
|
||||
Filters(query="one", place="one"),
|
||||
REGIONS={"HovedstadenOgBornholm": "Hovedstaden og Bornholm",
|
||||
"Midtjylland": "Midtjylland"}),
|
||||
"jobbank": _stub("jobbank", "Akademikernes Jobbank (Denmark)",
|
||||
{"q": "query", "region": "place", "remote": "remote"},
|
||||
Filters(query="one", place="one", remote="one"),
|
||||
REGIONS={"2": "Storkøbenhavn", "8": "Østjylland"},
|
||||
REMOTE={"helt": "Fully remote", "delvist": "Partly remote"}),
|
||||
}
|
||||
for name, module in stubs.items():
|
||||
monkeypatch.setitem(readers.PROVIDERS, name, module)
|
||||
return stubs
|
||||
@@ -181,6 +181,18 @@ def test_keep_order_ranks():
|
||||
key=keep_order).uid == "a"
|
||||
|
||||
|
||||
def test_every_search_provider_loses_to_a_board(search_stubs):
|
||||
from app.dupes import is_search_host
|
||||
|
||||
assert all(is_search_host(h) for h in
|
||||
("jobindex", "adzuna", "platsbanken", "freehire", "jobnet", "jobbank"))
|
||||
assert not any(is_search_host(h) for h in ("greenhouse", "workday", "teamtailor", ""))
|
||||
board = cp("z", host="greenhouse", desc=False, posted="2026-09-09")
|
||||
for host in ("freehire", "jobnet", "jobbank"):
|
||||
search = cp("a", host=host, desc=True, posted="2026-01-01")
|
||||
assert min([search, board], key=keep_order) is board
|
||||
|
||||
|
||||
def test_a_row_without_a_town_joins_only_one_group():
|
||||
# "Germany" matches Berlin and Munich; joining both would make them one role.
|
||||
rows = [cp("a", place="Berlin"), cp("b", place="Munich"), cp("c", place="Germany")]
|
||||
|
||||
@@ -61,6 +61,21 @@ def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store)
|
||||
assert dup_of(store) == {board.uid: None, search.uid: board.uid}
|
||||
|
||||
|
||||
def test_a_freehire_copy_of_a_greenhouse_role_is_hidden_behind_the_board(
|
||||
store: Store, search_stubs: dict[str, object]) -> None:
|
||||
"""freehire aggregates other hosts' postings; the same role it lists from
|
||||
a board roledesk reads is one role, and the board's copy is the one kept
|
||||
even though freehire's is the one with a description."""
|
||||
board = role("board", company="Lunar", title="Head of Data", place="Copenhagen",
|
||||
board="greenhouse", body="", day=22)
|
||||
copy = role("copy", company="Lunar", title="Head of Data", place="Copenhagen, Denmark",
|
||||
board=None, provider="freehire", source="greenhouse", day=20)
|
||||
for item in (copy, board):
|
||||
store.upsert_role(item)
|
||||
store.rebuild_dupes()
|
||||
assert dup_of(store) == {board.uid: None, copy.uid: board.uid}
|
||||
|
||||
|
||||
def kept_of(store: Store, *items: Item) -> str:
|
||||
"""The uid of the one of these roles that is not a copy."""
|
||||
found = dup_of(store)
|
||||
|
||||
+71
-1
@@ -576,7 +576,7 @@ def test_search_params_keeps_only_what_the_provider_reads() -> None:
|
||||
def test_the_providers_and_their_params_follow_the_reader_declarations() -> None:
|
||||
from plugins import readers
|
||||
|
||||
assert sources.PROVIDERS == tuple(readers.PROVIDERS) == ("jobindex", "adzuna", "platsbanken")
|
||||
assert tuple(readers.PROVIDERS)[:3] == ("jobindex", "adzuna", "platsbanken")
|
||||
assert sources.search_params("adzuna", " backend ", country=" DE ", where=" Berlin ",
|
||||
area=1, region="x", pages=2) == {
|
||||
"q": "backend", "country": "de", "where": "Berlin", "pages": 2}
|
||||
@@ -589,3 +589,73 @@ def test_search_params_refuses_a_region_that_is_no_swedish_lan() -> None:
|
||||
sources.search_params("platsbanken", "data", region="zYpL_aGk_QY8") # Midland
|
||||
with pytest.raises(ValueError, match="provider"):
|
||||
sources.search_params("indeed", "data")
|
||||
|
||||
|
||||
def test_search_params_of_the_three_newer_providers_follow_their_declarations(
|
||||
search_stubs: dict[str, Any]) -> None:
|
||||
assert sources.search_params("freehire", " head of data ", country=" DK,se ",
|
||||
remote="remote", area=1, region="x", pages=2) == {
|
||||
"q": "head of data", "country": "dk,se", "remote": "remote", "pages": 2}
|
||||
assert sources.search_params("jobnet", "data", region="Midtjylland", country="dk",
|
||||
remote="remote") == {"q": "data", "region": "Midtjylland"}
|
||||
assert sources.search_params("jobbank", "data", region="2", remote="helt") == {
|
||||
"q": "data", "region": "2", "remote": "helt"}
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
sources.search_params("jobnet", "data", region="Fyn")
|
||||
with pytest.raises(ValueError, match="remote"):
|
||||
sources.search_params("freehire", "data", remote="helt")
|
||||
|
||||
|
||||
def test_the_provider_error_lists_the_loaded_readers(search_stubs: dict[str, Any]) -> None:
|
||||
with pytest.raises(ValueError, match=r"freehire.*jobnet.*jobbank"):
|
||||
sources.search_params("indeed", "data")
|
||||
|
||||
|
||||
def test_a_provider_table_is_empty_for_a_reader_that_is_not_loaded(
|
||||
search_stubs: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from plugins import readers
|
||||
|
||||
assert sources.provider_table("jobbank", "REMOTE") == {
|
||||
"helt": "Fully remote", "delvist": "Partly remote"}
|
||||
assert sources.provider_table("freehire", "REGIONS") == {} # it declares none
|
||||
monkeypatch.delitem(readers.PROVIDERS, "jobbank")
|
||||
assert sources.provider_table("jobbank", "REGIONS") == {}
|
||||
assert sources.provider_table("nobody", "MODES") == {}
|
||||
|
||||
|
||||
def test_cli_searches_add_takes_the_newer_providers_and_their_flags(
|
||||
search_stubs: dict[str, Any], store: Store, tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
monkeypatch.setattr(sources, "try_search", lambda config, provider, name, params: (
|
||||
[{"title": "Data lead", "company": "Acme", "place": "København",
|
||||
"url": "https://x.example/1"}], True))
|
||||
config = tmp_path / "config.yaml"
|
||||
config.write_text(f"database: {store.path}\n")
|
||||
|
||||
def add(*argv: str) -> int:
|
||||
return main(["-c", str(config), "searches", "add", *argv])
|
||||
|
||||
assert add("freehire", "Head of data", "--q", "head of data", "--country", "DK,SE",
|
||||
"--remote", "hybrid") == 0
|
||||
assert add("jobnet", "Data science", "--q", "data science", "--region",
|
||||
"HovedstadenOgBornholm") == 0
|
||||
assert add("jobbank", "Data director", "--q", "data director", "--region", "2",
|
||||
"--remote", "delvist") == 0
|
||||
capsys.readouterr()
|
||||
saved = {s.board: s.params for s in store.subscriptions(source="search")}
|
||||
assert saved == {
|
||||
"freehire": {"q": "head of data", "country": "dk,se", "remote": "hybrid"},
|
||||
"jobnet": {"q": "data science", "region": "HovedstadenOgBornholm"},
|
||||
"jobbank": {"q": "data director", "region": "2", "remote": "delvist"}}
|
||||
assert add("jobnet", "Bad", "--q", "x", "--region", "Fyn") == 1
|
||||
assert "region" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_cli_searches_add_help_names_the_regions_and_work_modes(
|
||||
search_stubs: dict[str, Any], capsys: pytest.CaptureFixture[str]) -> None:
|
||||
with pytest.raises(SystemExit):
|
||||
main(["searches", "add", "-h"])
|
||||
text = " ".join(capsys.readouterr().out.split())
|
||||
assert "freehire" in text and "jobnet" in text and "jobbank" in text
|
||||
assert "HovedstadenOgBornholm" in text and "2 Storkøbenhavn" in text
|
||||
assert "hybrid Hybrid" in text and "helt Fully remote" in text
|
||||
@@ -1357,6 +1357,57 @@ def test_a_search_is_tried_saved_updated_and_removed(
|
||||
assert again["source"]["id"] == sub_id and again["source"]["on"] is True
|
||||
|
||||
|
||||
def test_the_newer_providers_save_with_their_own_params(
|
||||
client: TestClient, store: Store, search_stubs: dict[str, Any],
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_fake_search(monkeypatch, 2)
|
||||
freehire = client.post("/api/searches", json={
|
||||
"provider": "freehire", "name": "Head of data", "q": "head of data",
|
||||
"country": "DK,se", "remote": "hybrid", "area": 1, "region": "x"}).json()
|
||||
assert freehire["source"]["target"] == "freehire:head-of-data"
|
||||
assert freehire["source"]["params"] == {"q": "head of data", "country": "dk,se",
|
||||
"remote": "hybrid"}
|
||||
jobnet = client.post("/api/searches/try", json={
|
||||
"provider": "jobnet", "q": "data", "region": "Midtjylland", "remote": "helt"})
|
||||
assert jobnet.status_code == 200 and jobnet.json()["found"] == 2
|
||||
jobbank = client.post("/api/searches", json={
|
||||
"provider": "jobbank", "name": "Data director", "q": "data director", "region": "2",
|
||||
"remote": "helt"}).json()
|
||||
assert jobbank["source"]["params"] == {"q": "data director", "region": "2", "remote": "helt"}
|
||||
refused = client.post("/api/searches", json={
|
||||
"provider": "jobnet", "name": "Bad", "q": "data", "region": "Fyn"})
|
||||
assert refused.status_code == 422 and "region" in refused.json()["detail"]
|
||||
assert [s.board for s in store.subscriptions(source="search")] == ["freehire", "jobbank"]
|
||||
|
||||
|
||||
def test_the_sources_form_offers_the_providers_that_are_loaded(
|
||||
client: TestClient, search_stubs: dict[str, Any]) -> None:
|
||||
page = client.get("/settings/sources").text
|
||||
for option in ('<option value="freehire">freehire (tech, many countries)</option>',
|
||||
'<option value="jobnet">Jobnet (Denmark)</option>',
|
||||
'<option value="jobbank">Akademikernes Jobbank (Denmark)</option>',
|
||||
'<option value="HovedstadenOgBornholm">Hovedstaden og Bornholm</option>',
|
||||
'<option value="2">Storkøbenhavn</option>',
|
||||
'<option value="hybrid">Hybrid</option>',
|
||||
'<option value="delvist">Partly remote</option>'):
|
||||
assert option in page
|
||||
for control in ('id="s-countries"', 'id="s-mode"', 'id="s-region-jobnet"',
|
||||
'id="s-region-jobbank"', 'id="s-remote-jobbank"', 'id="s-region"'):
|
||||
assert page.count(control) == 1
|
||||
assert 'value="jobindex"' in page and 'value="platsbanken"' in page
|
||||
|
||||
|
||||
def test_the_sources_form_renders_without_the_newer_providers(
|
||||
client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from plugins import readers
|
||||
|
||||
for name in ("freehire", "jobnet", "jobbank"):
|
||||
monkeypatch.delitem(readers.PROVIDERS, name, raising=False)
|
||||
page = client.get("/settings/sources").text
|
||||
assert 'value="adzuna"' in page and 'value="freehire"' not in page
|
||||
assert 'id="s-region-jobnet"' in page and 'value="HovedstadenOgBornholm"' not in page
|
||||
|
||||
|
||||
def test_a_search_that_finds_nothing_or_is_malformed_is_refused(
|
||||
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_fake_search(monkeypatch, 0)
|
||||
|
||||
Reference in new issue
Block a user