Adds the search providers' PARAMS, clean_params and company, and a test that the example READS is workday.py's. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
593 lines
27 KiB
Python
593 lines
27 KiB
Python
"""The reader contract, held for every module of `plugins/readers/`.
|
|
|
|
Per module: it declares `READS`, the declaration matches its functions, and
|
|
its rows state what the declaration says and nothing it denies. No network:
|
|
fixtures stand in for the host. A fixture a declaration needs and that is
|
|
missing fails the test with the path to add.
|
|
|
|
Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/`
|
|
for searches, named after the module:
|
|
|
|
- `<name>-places.<ext>`: one listing answer, required of every reader. For
|
|
`listing="whole"` it is the body `parse` gets; for `listing="paged"` it is
|
|
what every request of the first `page` call returns, and `rows` reads
|
|
that page. Under `places="many"` at least one role is at two or more
|
|
places.
|
|
- `<name>.<ext>` and `<name>-teaser.<ext>`: more listing answers, read the
|
|
same way; the teaser one is required under `text="excerpt"` or
|
|
`teaser=True`.
|
|
- `<name>-detail*.<ext>` and `<name>-places-detail.<ext>`: what `detail`'s
|
|
requests return, one of them required under `text="detail"`. Only
|
|
`_runner.DETAIL_FIELDS` of the answer count, as at run time; the places of
|
|
`-places-detail` replace those of the `-places` rows.
|
|
- `<name>.slug`: optional, the slug the fixtures are read with, one line;
|
|
`acme` without it. A search's saved params are `SEARCH_PARAMS` below.
|
|
|
|
Between them the listing and detail answers show every field the reader
|
|
declares `stated` (or `figures`, `line`, `trusted`, `listed`): a real
|
|
fixture that happens to lack one is filled from what the per-host test
|
|
already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import textwrap
|
|
import types
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pydantic
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.prep.hosts import TABLES # noqa: E402
|
|
from plugins import readers # noqa: E402
|
|
from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402
|
|
from plugins.readers import _contract # noqa: E402
|
|
from plugins.readers._runner import DETAIL_FIELDS # noqa: E402
|
|
|
|
BOARD_FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
SEARCH_FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
|
|
|
#: A search's saved params, which its `page` reads; `{}` for the rest.
|
|
SEARCH_PARAMS: dict[str, dict[str, Any]] = {"adzuna": {"q": "backend", "country": "de"}}
|
|
|
|
#: The readers whose listing states no date and whose `posted` comes from
|
|
#: `detail` alone: the date exists only for roles the enrich stage read.
|
|
POSTED_BY_DETAIL = frozenset({"bamboohr", "gem", "rippling"})
|
|
|
|
#: The fields a host states in its own words, which `app.prep` maps through
|
|
#: the host's table in `app/prep/hosts/<name>.py`.
|
|
WORD_FIELDS = ("seniority", "employment")
|
|
|
|
|
|
def _folder(name: str) -> Path:
|
|
return SEARCH_FIXTURES if name in readers.PROVIDERS else BOARD_FIXTURES
|
|
|
|
|
|
def _fixtures(name: str, pattern: str) -> list[Path]:
|
|
return sorted(p for p in _folder(name).glob(pattern.format(name=name))
|
|
if p.suffix not in (".slug",))
|
|
|
|
|
|
def _fixture(name: str, kind: str) -> Path | None:
|
|
found = _fixtures(name, f"{{name}}-{kind}.*")
|
|
return found[0] if found else None
|
|
|
|
|
|
def _need(name: str, kind: str, why: str) -> Path:
|
|
"""The fixture `<name>-<kind>.*`, or a failure naming the file to add."""
|
|
found = _fixture(name, kind)
|
|
if found is None:
|
|
folder = _folder(name).relative_to(ROOT)
|
|
pytest.fail(f"add {folder}/{name}-{kind}.<ext>: {why}")
|
|
return found
|
|
|
|
|
|
def _slug(name: str) -> str:
|
|
path = _folder(name) / f"{name}.slug"
|
|
return path.read_text(encoding="utf-8").strip() if path.exists() else "acme"
|
|
|
|
|
|
def _answer(body: bytes) -> Callable[..., bytes]:
|
|
return lambda url, **kwargs: body
|
|
|
|
|
|
def read_fixture(module: types.ModuleType, body: bytes, slug: str,
|
|
detail_body: bytes | None = None, ask: Ask | None = None) -> list[dict[str, Any]]:
|
|
"""The rows `module` makes of one listing answer, as the runner reads
|
|
them: `parse` for a whole listing, the first `page` and its `rows` for a
|
|
paged one, and, given `detail_body`, each row's `places` from `detail`."""
|
|
reads: Reads = module.READS
|
|
if reads.listing == "whole":
|
|
rows = list(module.parse(body, slug))
|
|
else:
|
|
decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, ask or Ask())
|
|
rows = list(module.rows(decoded, slug))
|
|
if detail_body is not None:
|
|
for row in rows:
|
|
found = module.detail(_answer(detail_body), slug, row)
|
|
if "places" in found:
|
|
row["places"] = found["places"]
|
|
return rows
|
|
|
|
|
|
def _ask(name: str) -> Ask:
|
|
return Ask(params=SEARCH_PARAMS.get(name, {}))
|
|
|
|
|
|
def listing_rows(name: str) -> list[dict[str, Any]]:
|
|
"""Every row of the reader's listing fixtures; the `-places` rows with
|
|
the places of `-places-detail` where there is one."""
|
|
module, slug = readers.READERS[name], _slug(name)
|
|
places = _need(name, "places", "one listing answer of the host")
|
|
detail = _fixture(name, "places-detail")
|
|
rows = read_fixture(module, places.read_bytes(), slug,
|
|
detail.read_bytes() if detail else None, _ask(name))
|
|
for path in _fixtures(name, "{name}.*") + _fixtures(name, "{name}-teaser.*"):
|
|
rows += read_fixture(module, path.read_bytes(), slug, ask=_ask(name))
|
|
return rows
|
|
|
|
|
|
def detail_rows(name: str, listed: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""What `detail` hands over for the first listed role, once per detail
|
|
fixture, cut to `DETAIL_FIELDS` as the runner cuts it."""
|
|
module = readers.READERS[name]
|
|
if module.READS.text != "detail":
|
|
return []
|
|
paths = _fixtures(name, "{name}-detail*.*") + _fixtures(name, "{name}-places-detail.*")
|
|
if not paths:
|
|
_need(name, "detail", "what the role's own page answers, for text=\"detail\"")
|
|
assert listed, f"{name}: the listing fixtures gave no role to read in detail"
|
|
return [{key: value for key, value in
|
|
module.detail(_answer(path.read_bytes()), _slug(name), dict(listed[0])).items()
|
|
if key in DETAIL_FIELDS} for path in paths]
|
|
|
|
|
|
def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
|
|
"""What rows get wrong about `places` under the declared trait."""
|
|
out = []
|
|
for row in rows:
|
|
found = row.get("places")
|
|
rid = row.get("source_id")
|
|
if not isinstance(found, list):
|
|
out.append(f"{rid}: places is not a list")
|
|
continue
|
|
for place in found:
|
|
if not (isinstance(place, dict) and set(place) == {"text", "remote"}
|
|
and isinstance(place["text"], str) and place["text"].strip()
|
|
and place["remote"] in (True, False, None)):
|
|
out.append(f"{rid}: {place!r} is not a Place")
|
|
if places in ("one", "copies") and len(found) > 1:
|
|
out.append(f'{rid}: {len(found)} places under places="{places}"')
|
|
if places == "none" and found:
|
|
out.append(f'{rid}: {len(found)} places under places="none"')
|
|
if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows):
|
|
out.append('no row has two places: a reader with places="many" keeps every place')
|
|
elif places != "none" and not any(row.get("places") for row in rows):
|
|
out.append(f'no row has a place under places="{places}": declare "none"')
|
|
return out
|
|
|
|
|
|
def _states(rows: list[dict[str, Any]], key: str) -> bool:
|
|
"""Whether some row carries `key` with a value (False counts, "" does not)."""
|
|
return any(row.get(key) not in (None, "", [], {}) for row in rows)
|
|
|
|
|
|
def field_problems(reads: Reads, listing: list[dict[str, Any]], detail: list[dict[str, Any]],
|
|
posted_by_detail: bool = False) -> list[str]:
|
|
"""What the rows get wrong about the declared `fields`, `text` and
|
|
`teaser`: a trait that promises a key no row carries, or denies one a
|
|
row carries."""
|
|
rows = listing + detail
|
|
fields = reads.fields
|
|
out: list[str] = []
|
|
figures = _states(rows, "salary_min") or _states(rows, "salary_max")
|
|
line = _states(rows, "salary_text")
|
|
if fields.pay == "figures" and not figures:
|
|
out.append('pay="figures" and no row has salary_min or salary_max')
|
|
if fields.pay == "line" and (figures or not line):
|
|
out.append('pay="line" wants salary_text on some row and figures on none')
|
|
if fields.pay == "none" and (figures or line):
|
|
out.append('pay="none" and a row has pay')
|
|
|
|
if fields.posted == "none" and _states(rows, "posted"):
|
|
out.append('posted="none" and a row has posted')
|
|
elif fields.posted != "none" and posted_by_detail:
|
|
if _states(listing, "posted") or not _states(detail, "posted"):
|
|
out.append(f'posted="{fields.posted}" by detail wants posted from detail only')
|
|
elif fields.posted != "none" and not _states(listing, "posted"):
|
|
out.append(f'posted="{fields.posted}" and no listed row has posted')
|
|
|
|
for key in ("remote", "country", "seniority", "employment"):
|
|
stated = getattr(fields, key) == "stated"
|
|
if stated != _states(rows, key):
|
|
out.append(f'{key}="{getattr(fields, key)}" and '
|
|
+ ("no row has it" if stated else "a row has it"))
|
|
|
|
text = {"listing": _states(listing, "description"),
|
|
"detail": _states(detail, "description"),
|
|
"excerpt": _states(listing, "teaser") and not _states(listing, "description"),
|
|
"none": not _states(rows, "description")}[reads.text]
|
|
if not text:
|
|
out.append(f'text="{reads.text}" and the rows do not show it')
|
|
teaser = reads.teaser or reads.text == "excerpt"
|
|
if teaser != _states(listing if teaser else rows, "teaser"):
|
|
out.append("teaser declared and no listed row has one" if teaser
|
|
else "a row has a teaser the declaration does not name")
|
|
return out
|
|
|
|
|
|
# ------------------------------------------------------------ every module
|
|
|
|
@pytest.mark.parametrize("name", readers.READERS)
|
|
def test_a_declaration_matches_the_functions(name: str) -> None:
|
|
module = readers.READERS[name]
|
|
assert readers.problems(module, name) == []
|
|
assert name in (readers.PROVIDERS if module.READS.scope == "search" else readers.HOSTS)
|
|
|
|
|
|
@pytest.mark.parametrize("name", readers.READERS)
|
|
def test_a_reader_keeps_every_place(name: str) -> None:
|
|
listed = listing_rows(name)
|
|
assert listed, f"{name}: the listing fixtures gave no rows"
|
|
answered = [{"source_id": f"detail {i}", "places": row["places"]}
|
|
for i, row in enumerate(detail_rows(name, listed)) if "places" in row]
|
|
assert place_problems(listed + answered, readers.READERS[name].READS.places) == []
|
|
|
|
|
|
@pytest.mark.parametrize("name", readers.READERS)
|
|
def test_a_reader_states_what_it_declares(name: str) -> None:
|
|
reads = readers.READERS[name].READS
|
|
if reads.text == "excerpt" or reads.teaser:
|
|
_need(name, "teaser", "a listing answer with a teaser")
|
|
listed = listing_rows(name)
|
|
assert field_problems(reads, listed, detail_rows(name, listed),
|
|
name in POSTED_BY_DETAIL) == []
|
|
|
|
|
|
@pytest.mark.parametrize("name", [name for name, module in readers.READERS.items()
|
|
if module.READS.listing == "paged"
|
|
and module.READS.complete in ("stated", "end")])
|
|
def test_a_paged_reader_states_a_total_as_declared(name: str) -> None:
|
|
"""`complete="stated"`: the first page's total is an int; `"end"`: None,
|
|
since the read ends on an empty page."""
|
|
module = readers.READERS[name]
|
|
body = _need(name, "places", "one listing answer of the host").read_bytes()
|
|
_decoded, _results, total, _cursor = module.page(_answer(body), _slug(name), None,
|
|
_ask(name))
|
|
if module.READS.complete == "stated":
|
|
assert isinstance(total, int) and not isinstance(total, bool), total
|
|
else:
|
|
assert total is None, total
|
|
|
|
|
|
def test_the_posted_by_detail_list_names_only_readers_that_date_by_detail() -> None:
|
|
for name in POSTED_BY_DETAIL:
|
|
reads = readers.READERS[name].READS
|
|
assert reads.text == "detail" and reads.fields.posted != "none", name
|
|
|
|
|
|
# ------------------------------------------------------------ prep tables
|
|
|
|
def test_every_prep_table_is_a_readers() -> None:
|
|
assert set(TABLES) <= set(readers.READERS), set(TABLES) - set(readers.READERS)
|
|
|
|
|
|
@pytest.mark.parametrize("name", readers.READERS)
|
|
def test_a_reader_stating_words_has_a_table_for_them(name: str) -> None:
|
|
"""A host that states seniority or employment in its own words has a
|
|
table for that field in `app/prep/hosts/<name>.py`, even an empty one
|
|
for a host whose words are already the app's (`words.lookup` then
|
|
takes the word as it is). The table is where a word is known, so a
|
|
host without one lists every word as unmapped. A table for a field the
|
|
reader declares `none` is dead."""
|
|
fields = readers.READERS[name].READS.fields
|
|
tables = TABLES.get(name, {})
|
|
for field in WORD_FIELDS:
|
|
has = field in tables or (field == "employment" and "match" in tables)
|
|
stated = getattr(fields, field) == "stated"
|
|
assert has == stated, (f"{name}: {field}={getattr(fields, field)!r} "
|
|
f"{'without' if stated else 'with'} a table in "
|
|
f"app/prep/hosts/{name}.py")
|
|
if "remote" in tables:
|
|
assert fields.remote == "stated", f"{name}: a remote table and remote=\"none\""
|
|
|
|
|
|
# ------------------------------------------------------------ the checks
|
|
|
|
def _reader(**functions: Any) -> Any:
|
|
module: Any = types.ModuleType("fake")
|
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", (r"fake\.example/([a-z]+)",)
|
|
module.slug_ok = lambda slug: True
|
|
module.allowed = lambda slug: lambda host: host == "fake.example"
|
|
module.careers_url = lambda slug: "https://fake.example/" + slug
|
|
for name, function in functions.items():
|
|
setattr(module, name, function)
|
|
return module
|
|
|
|
|
|
def _whole(**functions: Any) -> Any:
|
|
return _reader(**{"feed_url": lambda slug, ask: "https://fake.example/" + slug,
|
|
"parse": lambda body, slug: []} | functions)
|
|
|
|
|
|
def _paged(**functions: Any) -> Any:
|
|
return _reader(**{"page": lambda get, slug, cursor, ask: (None, [], None, None),
|
|
"rows": lambda decoded, slug: []} | functions)
|
|
|
|
|
|
_ROBOTS = "read: no robots.txt rule on /api/ (checked 2026-09-30)"
|
|
|
|
|
|
def _reads(**traits: Any) -> Reads:
|
|
base: dict[str, Any] = {"scope": "board", "listing": "whole", "text": "listing",
|
|
"complete": "whole", "places": "one", "robots": _ROBOTS}
|
|
return Reads(**(base | traits))
|
|
|
|
|
|
def test_a_good_declaration_has_no_problems() -> None:
|
|
module = _whole()
|
|
module.READS = _reads()
|
|
assert readers.problems(module, "fake") == []
|
|
module = _paged(detail=lambda get, slug, row: {},
|
|
facets=lambda get, slug: {"country": {"DE": "1"}})
|
|
module.READS = _reads(listing="paged", page_size=20, ceiling=2000, text="detail",
|
|
teaser=True, complete="stated", filters=Filters(country="ids"),
|
|
places="many", fields=Fields(posted="listed", remote="stated"),
|
|
empty="suspect", title="provisional")
|
|
assert readers.problems(module, "fake") == []
|
|
|
|
|
|
def test_a_module_without_reads_fails() -> None:
|
|
module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: [])
|
|
assert readers.problems(module, "fake") == [
|
|
"no READS: every reader declares what it reads"]
|
|
|
|
|
|
@pytest.mark.parametrize(("module", "reads", "problem"), [
|
|
(_reader(feed_url=lambda slug, ask: "", parse=lambda b, s: []),
|
|
{"listing": "paged", "complete": "stated"}, 'listing="paged" without page and rows'),
|
|
(_paged(), {}, 'listing="whole" without feed_url and parse'),
|
|
(_whole(), {"complete": "stated"}, 'complete="whole" goes with listing="whole"'),
|
|
(_whole(), {"page_size": 20}, 'page_size and ceiling are for listing="paged"'),
|
|
(_paged(), {"listing": "paged", "complete": "end", "page_size": 100, "ceiling": 50},
|
|
"ceiling is below page_size"),
|
|
(_paged(fallback_url=lambda slug: ""), {"listing": "paged", "complete": "end"},
|
|
'fallback_url without listing="whole"'),
|
|
(_whole(), {"text": "detail"}, 'text="detail" without detail'),
|
|
(_whole(detail=lambda g, s, r: {}), {}, 'detail with text="listing"'),
|
|
(_whole(), {"teaser": True, "text": "excerpt"}, "teaser is for text=\"detail\""),
|
|
(_whole(), {"title": "provisional"}, 'title="provisional" without detail'),
|
|
(_whole(), {"filters": Filters(country="ids")}, "an ids filter (country) without facets"),
|
|
(_whole(facets=lambda g, s: {}), {"filters": Filters(country="one")},
|
|
"facets without an ids filter"),
|
|
(_whole(facets=lambda g, s: {}), {"filters": Filters(query="ids")},
|
|
'query cannot be "ids"'),
|
|
(_paged(), {"scope": "search", "listing": "paged", "complete": "end"},
|
|
'scope="search" needs complete="never" or "stated"'),
|
|
(_paged(), {"listing": "paged", "complete": "never"},
|
|
'scope="board" cannot be complete="never"'),
|
|
(_whole(), {"key": Key(name="Adzuna Key", fields=("app_id",))},
|
|
"key names no config entry"),
|
|
(_whole(), {"key": Key(name="adzuna", fields=())}, "key has no fields"),
|
|
(_whole(), {"key": Key(name="adzuna", fields=("app_id", "app_key"),
|
|
env=("ADZUNA_APP_ID",))},
|
|
"key.env does not match key.fields"),
|
|
(_whole(), {"robots": " "}, "robots is empty"),
|
|
(_whole(), {"robots": "read: robots.txt allows /api/"}, "robots has no date"),
|
|
(_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {},
|
|
"feed_url does not take (slug, ask)"),
|
|
(_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []),
|
|
{"listing": "paged", "complete": "end"}, "page does not take (get, slug, cursor, ask)"),
|
|
])
|
|
def test_the_declaration_check_names_each_problem(
|
|
module: Any, reads: dict[str, Any], problem: str) -> None:
|
|
module.READS = _reads(**reads)
|
|
found = readers.problems(module, "fake")
|
|
assert any(p.startswith(problem) for p in found), found
|
|
|
|
|
|
def test_reads_that_is_not_a_reads_is_refused() -> None:
|
|
module = _whole()
|
|
module.READS = {"scope": "board"}
|
|
assert "READS is not a Reads" in readers.problems(module, "fake")
|
|
|
|
|
|
def _search() -> Any:
|
|
module: Any = types.ModuleType("fake")
|
|
module.NAME, module.LABEL = "fake", "Fake"
|
|
module.allowed = lambda slug: lambda host: True
|
|
module.page = lambda get, slug, cursor, ask: (None, [], 0, None)
|
|
module.rows = lambda decoded, slug: []
|
|
module.PARAMS = {"q": "query", "where": "place"}
|
|
module.clean_params = lambda params: dict(params)
|
|
module.READS = _reads(scope="search", listing="paged", complete="never",
|
|
filters=Filters(query="one", place="one"),
|
|
key=Key(name="adzuna", fields=("app_id", "app_key"),
|
|
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")))
|
|
return module
|
|
|
|
|
|
def test_a_search_needs_fewer_names_than_a_board() -> None:
|
|
module = _search()
|
|
assert readers.problems(module, "fake") == []
|
|
del module.READS
|
|
assert "no careers_url" in readers.problems(module, "fake")
|
|
|
|
|
|
def test_a_search_names_its_params_and_cleans_them() -> None:
|
|
module = _search()
|
|
del module.PARAMS, module.clean_params
|
|
assert {"no PARAMS", "no clean_params"} <= set(readers.problems(module, "fake"))
|
|
module = _search()
|
|
module.PARAMS = {"q": "query", "area": "town"}
|
|
assert readers.problems(module, "fake") == [
|
|
"PARAMS maps 'area' to 'town', not a filter dimension"]
|
|
module = _search()
|
|
module.clean_params = lambda: {}
|
|
assert readers.problems(module, "fake") == ["clean_params does not take (params)"]
|
|
|
|
|
|
def test_every_search_names_its_params() -> None:
|
|
for name, module in readers.PROVIDERS.items():
|
|
declared = module.READS.filters.declared()
|
|
assert "query" in module.PARAMS.values(), name
|
|
assert set(module.PARAMS.values()) <= set(declared), name
|
|
|
|
|
|
def test_a_misspelt_trait_fails_where_it_is_declared() -> None:
|
|
with pytest.raises(pydantic.ValidationError):
|
|
_reads(listing="pages")
|
|
with pytest.raises(pydantic.ValidationError):
|
|
Filters(country="some")
|
|
with pytest.raises(pydantic.ValidationError):
|
|
_reads(robots=None)
|
|
with pytest.raises(pydantic.ValidationError):
|
|
Reads(scope="board", listing="whole", text="listing", complete="whole",
|
|
places="one") # type: ignore[call-arg] # robots is required
|
|
with pytest.raises(pydantic.ValidationError):
|
|
_reads(colour="blue") # extra keys are refused
|
|
|
|
|
|
def test_a_declaration_is_frozen() -> None:
|
|
reads = _reads()
|
|
with pytest.raises(pydantic.ValidationError):
|
|
reads.places = "many" # type: ignore[misc]
|
|
|
|
|
|
def test_the_loader_refuses_a_module_whose_declaration_breaks_the_contract(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
package = tmp_path / "readersbad"
|
|
package.mkdir()
|
|
(package / "__init__.py").write_text("")
|
|
(package / "odd.py").write_text(
|
|
"from plugins.readers import Reads\n"
|
|
"NAME = 'odd'\nLABEL = 'Odd'\nPATTERNS = ()\n"
|
|
"def slug_ok(slug): return True\n"
|
|
"def allowed(slug): return lambda h: True\n"
|
|
"def careers_url(slug): return ''\n"
|
|
"def feed_url(slug, ask): return ''\n"
|
|
"def parse(body, slug): return []\n"
|
|
"READS = Reads(scope='board', listing='whole', text='detail', complete='whole',\n"
|
|
" places='many', robots='read (checked 2026-09-30)')\n")
|
|
monkeypatch.syspath_prepend(str(tmp_path))
|
|
with pytest.raises(ImportError, match=r'readersbad\.odd .*text="detail" without detail'):
|
|
readers.load([str(package)], "readersbad")
|
|
|
|
|
|
def test_filters_list_what_is_filtered() -> None:
|
|
assert Filters(country="one", query="many").declared() == {"country": "one",
|
|
"query": "many"}
|
|
|
|
|
|
# ------------------------------------------------------------ the fixture check
|
|
|
|
def _fake_many(rows: list[dict[str, Any]]) -> Any:
|
|
module = _reader(feed_url=lambda slug, ask: "", parse=lambda body, slug: rows)
|
|
module.READS = _reads(places="many")
|
|
return module
|
|
|
|
|
|
def test_the_place_check_passes_a_role_at_two_places() -> None:
|
|
rows = [{"source_id": "1", "places": [{"text": "Berlin", "remote": False},
|
|
{"text": "Remote (DE)", "remote": True}]}]
|
|
assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == []
|
|
|
|
|
|
def test_the_place_check_holds_a_reader_that_states_none_to_none() -> None:
|
|
assert place_problems([{"source_id": "1", "places": []}], "none") == []
|
|
assert place_problems([{"source_id": "1", "places": []}], "one") == [
|
|
'no row has a place under places="one": declare "none"']
|
|
assert place_problems([{"source_id": "2", "places": [{"text": "Oslo", "remote": None}]}],
|
|
"none") == ['2: 1 places under places="none"']
|
|
|
|
|
|
def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None:
|
|
joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}]
|
|
assert place_problems(joined, "many") == [
|
|
'no row has two places: a reader with places="many" keeps every place']
|
|
assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [
|
|
"2: places is not a list", 'no row has a place under places="one": declare "none"']
|
|
assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [
|
|
"3: ('Berlin', False) is not a Place"]
|
|
|
|
|
|
def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
|
|
seen: list[bytes] = []
|
|
|
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
|
seen.append(get("https://fake.example/list"))
|
|
return ([{"id": "1"}], [{"id": "1"}], 1, None)
|
|
|
|
module = _paged(page=page, rows=lambda decoded, slug: [{"source_id": d["id"], "places": []}
|
|
for d in decoded],
|
|
detail=lambda get, slug, row: {"places": [
|
|
{"text": get("https://fake.example/1").decode(), "remote": None},
|
|
{"text": "Munich", "remote": None}]})
|
|
module.READS = _reads(listing="paged", complete="stated", text="detail", places="many")
|
|
rows = read_fixture(module, b"Berlin", "acme", detail_body=b"Berlin")
|
|
assert seen == [b"Berlin"]
|
|
assert place_problems(rows, "many") == []
|
|
|
|
|
|
def test_the_contracts_example_is_workdays_declaration() -> None:
|
|
doc = readers.__doc__ or ""
|
|
start = doc.index(" READS = Reads(")
|
|
source = textwrap.dedent(doc[start:doc.index("\n )\n", start) + 6])
|
|
namespace: dict[str, Any] = {"Reads": Reads, "Filters": Filters, "Fields": Fields}
|
|
exec(source, namespace)
|
|
assert namespace["READS"] == readers.READERS["workday"].READS
|
|
|
|
|
|
def test_the_contract_types_are_what_the_package_exports() -> None:
|
|
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
|
|
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
|
|
|
|
|
|
# ------------------------------------------------------------ the field check
|
|
|
|
def test_the_field_check_passes_rows_that_state_what_is_declared() -> None:
|
|
reads = _reads(text="detail", teaser=True, listing="whole",
|
|
fields=Fields(pay="figures", posted="trusted", remote="stated"))
|
|
listed = [{"source_id": "1", "teaser": "Short.", "remote": False, "posted": "2026-09-30"}]
|
|
detail = [{"description": "Long.", "salary_min": 50000}]
|
|
assert field_problems(reads, listed, detail) == []
|
|
|
|
|
|
def test_the_field_check_names_a_promise_no_row_keeps_and_a_denial_a_row_breaks() -> None:
|
|
reads = _reads(fields=Fields(pay="line", employment="stated"))
|
|
listed = [{"source_id": "1", "description": "Text.", "salary_min": 1, "remote": "hybrid",
|
|
"posted": "2026-09-30"}]
|
|
assert field_problems(reads, listed, []) == [
|
|
'pay="line" wants salary_text on some row and figures on none',
|
|
'posted="none" and a row has posted',
|
|
'remote="none" and a row has it',
|
|
'employment="stated" and no row has it']
|
|
|
|
|
|
def test_the_field_check_takes_a_date_from_detail_only_where_it_is_said() -> None:
|
|
reads = _reads(text="detail", fields=Fields(posted="trusted"))
|
|
listed = [{"source_id": "1"}]
|
|
detail = [{"description": "Text.", "posted": "2026-09-30"}]
|
|
assert field_problems(reads, listed, detail, posted_by_detail=True) == []
|
|
assert field_problems(reads, listed, detail) == [
|
|
'posted="trusted" and no listed row has posted']
|
|
assert field_problems(reads, [*listed, {"posted": "2026-09-30"}], detail,
|
|
posted_by_detail=True) == [
|
|
'posted="trusted" by detail wants posted from detail only']
|
|
|
|
|
|
def test_the_field_check_holds_text_and_teaser_to_the_declaration() -> None:
|
|
excerpt = _reads(text="excerpt")
|
|
assert field_problems(excerpt, [{"teaser": "Short."}], []) == []
|
|
assert field_problems(excerpt, [{"teaser": "Short.", "description": "Long."}], []) == [
|
|
'text="excerpt" and the rows do not show it']
|
|
assert field_problems(_reads(), [{"description": "Long.", "teaser": "Short."}], []) == [
|
|
"a row has a teaser the declaration does not name"]
|