diff --git a/plugins/README.md b/plugins/README.md index 6cb4041..50951c6 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -1,6 +1,6 @@ # Where roles come from -This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a host module is the docstring of `plugins/readers/__init__.py`, not this file. +This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a reader (a host module or a search provider) is the docstring of `plugins/readers/__init__.py`, with its types and the declaration check in `plugins/readers/_contract.py` and the conformance test in `tests/test_readers_contract.py`, not this file. The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/readers/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`. diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py index 33148b9..29021ff 100644 --- a/plugins/readers/__init__.py +++ b/plugins/readers/__init__.py @@ -1,18 +1,21 @@ -"""The hosts the `ats` plugin reads, one module each. +"""The readers: one module per source of roles, a host's boards or a search. -A host registers by existing: every module in this package whose name does +A reader registers by existing: every module in this package whose name does not start with `_` is loaded when the package is imported, checked against -the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A -module that breaks the contract fails the import with an ImportError naming -the module and what it lacks, rather than a pull that fails board by board. -`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host -select and its description from `HOSTS`, and `app/jobs/discover.py` its -patterns and probes. The package sits beside `ats.py` rather than being -`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and -the tests patch names on the `ats` module. +the contract below, and listed in `READERS` in order of `(RANK, NAME)`; +`HOSTS` holds those of `scope="board"` (and every module without `READS`), +`PROVIDERS` those of `scope="search"`. A module that breaks the contract +fails the import with an ImportError naming the module and every problem, +rather than a pull that fails board by board. `plugins/ats.py` builds its +allow-list, feeds, pages, the Sources page's host select and its +description from `HOSTS`, and `app/jobs/discover.py` its patterns and +probes. The package sits beside `ats.py` rather than being `plugins/ats/`, +since `desk.sources.registry` loads `plugins/*.py` by path and the tests +patch names on the `ats` module. The types are in `._contract`. -A module is named after its host (`plugins/readers/workday.py`) and imports -`desk` and `._shared` only, never `plugins.ats`. It holds: +A module is named after its source (`plugins/readers/workday.py`) and +imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It +holds: NAME = "workday" # the host of a target `workday:`; equals the module's name LABEL = "Workday" # what the Sources page's select shows @@ -21,28 +24,87 @@ A module is named after its host (`plugins/readers/workday.py`) and imports KEEP_CASE = False # optional; True keeps a discovered slug's case ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case + READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000, + text="detail", complete="stated", + filters=Filters(country="ids"), places="many", + fields=Fields(posted="listed"), + robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)") + def slug_ok(slug: str) -> bool # strict, see below def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach def careers_url(slug: str) -> str # the page a person opens -and one of two ways to read a board: +and the functions its `READS` names: - def feed_url(slug: str) -> str # the whole board in one answer, + def feed_url(slug: str, ask: Ask) -> str # listing="whole": one answer, def parse(body: bytes, slug: str) -> list[Row] # read with `parse` - def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call, - def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page + def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged", + def rows(decoded: Any, slug: str) -> list[Row] # a page a call + + def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail" + def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter and, optionally: - def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read - # once when `feed_url` is over the cap - def robots_page(slug: str) -> str # the URL robots.txt must allow - def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page + def fallback_url(slug: str) -> str # listing="whole": a smaller feed, + # read once when `feed_url` is over the cap + def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt) def probe_url(slug: str) -> str # discovery: 2xx only for a board def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as +A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its +slug is the provider's own (a search reads no board), and it is not +discovered. A module without `READS` is read as a board with the older +signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by +`undeclared()`; the conformance test marks it until it declares. + +**Declaration.** `READS` states what the reader does; the runner does it. +`problems()` checks it against the functions: `paged` needs `page` and +`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`, +`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs +`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key` +names a config entry, `robots` is not empty, and each function takes the +arguments above. The traits: + +- `scope`: `board`, one company's whole list; `search`, many companies, + never a whole list, so nothing a search stops listing is closed by it. +- `listing`: `whole`, one request; `paged`, with `page_size` and `ceiling`, + the offset past which the host answers nothing whatever is asked. A board + above its ceiling is incomplete unless a filter brings it under. +- `text`: `listing`, the list has the full text; `excerpt`, a teaser and no + more; `detail`, one request a role; `none`. `teaser=True` with `detail` + says the list carries a teaser as well. +- `complete`: `stated`, the host gives a total; `end`, an empty page ends + the list; `whole`, the one answer is all; `never`. +- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`, + `one` (one value a request), `many` (a list in one request), `ids` (host + ids looked up with `facets`). +- `places`: `one`, `many`, or `copies` when the host publishes one posting + per place with its own id. +- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the + posting date; `listed`, a date the host moves; `none`), and `remote`, + `country`, `seniority`, `employment` (`stated` or `none`). +- `key`: `Key(name, fields, env)`, the config entry `sources.search.` + and its environment fallbacks. The runner resolves it and passes it in + `Ask.params["key"]`. +- `robots`: the robots.txt decision, see below. Required. +- `empty`: `suspect` when an unknown slug answers an empty list, so a read + far under the last count counts as failed. +- `title`: `provisional` when the listing may lack the title and `detail` + fills it. + +**Ask.** `Ask(countries, places, remote, query, params)` is what the runner +wants from one series of requests. It sets only dimensions the reader +declared. With `one` it runs a series per value and unions the rows by +`source_id`. With `ids`, `countries` and `places` hold host ids, which the +runner maps from `facets(get, slug)`: `{"country": {"DE": ""}, +"place": {...}}`, called at most once a day per board and kept in state. +`params` holds a search's saved params. `Ask()` is an unfiltered read. A +reader turns `Ask` into its host's query in `feed_url` or `page` and never +filters rows itself. + **Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts hold 10 to 80, which keeps the order they had before there were modules, so a new host comes after them unless it says otherwise. The order decides the @@ -60,10 +122,10 @@ with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`). group, the groups that matched joined by "/". Only group 1 is checked against discovery's words that name no board (`api`, `jobs`, `embed`, ...). The slug is lowercased unless `KEEP_CASE` is True: set it when the host's -paths are case-sensitive (Workday's site), since robots.txt paths are too. A -pattern without a group names the host with no slug. A host that -`discover.py` names in its own `_UNREADABLE` table is taken from its module -instead as soon as the module exists. +paths are case-sensitive (Workday's site). A pattern without a group names +the host with no slug. A host that `discover.py` names in its own +`_UNREADABLE` table is taken from its module instead as soon as the module +exists. **Fetching.** `allowed(slug)` returns a predicate over a lowercased host name; `_shared.only(host)` is the usual one. Every request goes through @@ -92,13 +154,32 @@ one without, when the read ended on an empty page. A 429 after the first page ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY` apart. A paged module may also have `parse(body, slug)` for one page's body. -**Rows.** A `Row` is a dict with `source_id` (the host's own id for the -role, unique on the board), `title`, `location`, `url` (http or https, -checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see -`_shared.parse_dt`), `department`, `description` (plain text, see -`desk.sources.strip_html`), `remote` (True, False, a word or None) and -`salary_text`; and, where the host states pay, `salary_min`, `salary_max` -(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`. +**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's +own id for the role, unique on the board), `title`, `url` (http or https, +checked with `_shared.is_web_url`) and `places`; `location` (the host's +one-line place, kept while storage reads it), `posted` (an aware datetime +or None, see `_shared.parse_dt`), `department`, `description` (plain text, +see `desk.sources.strip_html`), `remote` (True, False, a word or None) and +`salary_text`; where the host states pay, `salary_min`, `salary_max` +(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`; +where it states them, `country` (ISO 3166 alpha-2), `seniority` and +`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a +new field needs no change there. Undeclared modules do not state `places` +yet. + +**Places.** A role has a list of places: `places` is `[Place(text=..., +remote=...)]`, one entry per place the host states, in the host's order, in +the host's words. A reader never multiplies a row per place, never drops a +place and never joins places into one `text`. `remote` is True for a remote +place, False for an office, None when unsaid. A host that publishes one +posting per place, each with its own id, declares `places="copies"`: each +copy is a row with its one place, and `dupes.py` folds them into one role +that keeps each copy's link. + +**Teaser.** A short text the list carries (an excerpt, a summary) is handed +over as `teaser`, plain text, whether or not the full text comes later. It +is shown and embedded until `description` arrives. A reader that sees one +keeps it. **Detail.** `detail(get, slug, row)` reads one role's own page and returns a dict of the fields it found. The shared code calls it for roles not yet read @@ -117,18 +198,16 @@ A role read in detail on an earlier pull is sent without the listing's place and remote, so what its own page said about them stays. Completeness is decided by the listing alone. -**robots.txt.** With `robots_page(slug)`, the shared code reads -`https:///robots.txt` through `get` before the listing, at most -once a day per board (`ats:robots:`), following a redirect to the -host's `www.` twin or back and no other. The board fails with "robots.txt -disallows it" when the rules for `*` or for the app's own token disallow the -page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a -final `$` in paths, case-sensitive paths, or without case for a host with -`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as -Workday does). A 4xx other than 429 means no -rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails -the board with "robots.txt could not be read" and is asked again next pull. -`robots_page` must be on a host `allowed(slug)` admits. +**robots.txt.** Decided when the reader is built, not checked at run time. +Whoever builds a reader reads the host's robots.txt for the paths the reader +requests and reports what it disallows, and the owner decides whether the +host is read anyway. `robots` records that decision with its date, e.g. +`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is +disallowed: owner, 2026-09-30"`. The runtime check in `ats.py`, which +`robots_page` and `ROBOTS_ANY_CASE` feed, predates this rule and goes with +the runner; a new reader defines neither. Until then it reads +`https:///robots.txt` at most once a day per board and fails the board +when the page is disallowed for `*` or the app's token. **Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when the slug is a board; `probe_ok(body)` narrows that when a host answers an @@ -145,23 +224,41 @@ import re from collections.abc import Iterable from types import ModuleType +from . import _contract +from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row + +__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key", + "Place", "Reads", "Row", "load", "problems", "scope", "undeclared"] + #: The rank of a module that states none: after the eight first hosts, whose #: ranks (10 to 80) keep the order they had before there were modules. DEFAULT_RANK = 100 -#: What every host module defines. +#: What every board module defines. REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url") +#: What a module of `scope="search"` defines. +REQUIRED_SEARCH = ("NAME", "LABEL", "allowed") + #: The functions a module may define; each must be callable if present. FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows", - "fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs") + "fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok", + "probe_slugs") + + +def scope(module: ModuleType) -> str: + """`board` or `search`; `board` for a module without `READS`.""" + reads = getattr(module, "READS", None) + return reads.scope if isinstance(reads, Reads) else "board" def problems(module: ModuleType, name: str | None = None) -> list[str]: """What `module` lacks or gets wrong against the contract, one phrase each; empty for a module that keeps it. `name` is the module's own name - in the package, which `NAME` must equal.""" - out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)] + in the package, which `NAME` must equal. `READS` is checked when the + module declares one (`_contract.problems`).""" + required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED + out = [f"no {attr}" for attr in required if not hasattr(module, attr)] if name is not None and getattr(module, "NAME", name) != name: out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}") if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL): @@ -193,7 +290,7 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]: out.append("probe_ok without probe_url") if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"): out.append("probe_slugs without probe_url") - return out + return out + _contract.problems(module) def load(path: Iterable[str], package: str) -> dict[str, ModuleType]: @@ -206,12 +303,27 @@ def load(path: Iterable[str], package: str) -> dict[str, ModuleType]: continue module = importlib.import_module(f"{package}.{info.name}") if found := problems(module, info.name): - raise ImportError(f"{package}.{info.name} breaks the host contract: " + raise ImportError(f"{package}.{info.name} breaks the reader contract: " + "; ".join(found)) modules.append(module) modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME)) return {module.NAME: module for module in modules} -#: host -> its module, in order of `(RANK, NAME)`. -HOSTS: dict[str, ModuleType] = load(__path__, __name__) +def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]: + """The names of the modules (of `READERS` by default) without `READS`, + read with the older signatures until they declare.""" + return [name for name, module in (READERS if modules is None else modules).items() + if not hasattr(module, "READS")] + + +#: name -> module for every reader, in order of `(RANK, NAME)`. +READERS: dict[str, ModuleType] = load(__path__, __name__) + +#: host -> its module: the readers of `scope="board"` and the undeclared. +HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items() + if scope(module) == "board"} + +#: provider -> its module: the readers of `scope="search"`. +PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items() + if scope(module) == "search"} diff --git a/plugins/readers/_contract.py b/plugins/readers/_contract.py new file mode 100644 index 0000000..250d494 --- /dev/null +++ b/plugins/readers/_contract.py @@ -0,0 +1,291 @@ +"""The types of the reader contract, and the check of a module's declaration +against its functions. The prose is the package's docstring; this file is +what it names. + +A reader declares `READS = Reads(...)`: what it reads (a board or a search), +how the listing arrives, where the text is, how completeness is known, what +it can filter at the source, how it states places and which fields it +states. The runner reads the declaration and does the rest. Values are +checked by pydantic where the module is declared, so a misspelt trait fails +the import; what depends on the module's functions, or on two traits at +once, is `problems()`, which the package's loader runs and turns into an +ImportError naming every problem. + +Signatures of a module that declares `READS`, with `Get` and `Page` from +`._shared`: + + def feed_url(slug: str, ask: Ask) -> str # listing="whole" + def parse(body: bytes, slug: str) -> list[Row] # listing="whole" + def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged" + def rows(decoded: Any, slug: str) -> list[Row] # listing="paged" + def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail" + def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter + +A module without `READS` keeps the signatures without `ask` until it +declares; `undeclared()` in the package lists those modules. +""" + +from __future__ import annotations + +import inspect +import re +from collections.abc import Callable +from types import ModuleType +from typing import Any, Literal, Required, TypedDict + +from pydantic import ConfigDict + +from desk.models import Model + +#: A filter dimension's reach at the source: `none`, not filtered; `one`, one +#: value a request (the runner makes one series per value and unions them by +#: id); `many`, a list in one request; `ids`, values the host names by its own +#: ids, which `facets` looks up. +Filter = Literal["none", "one", "many", "ids"] + +#: Whether the reader states a field: `stated` from the host, `none` never. +Stated = Literal["stated", "none"] + +#: A config entry name: `sources.search.` in `config.yaml`. +_CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*") + +#: An environment variable name. +_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*") + + +class _Frozen(Model): + """A declaration: immutable, extra keys refused.""" + + model_config = ConfigDict(from_attributes=True, extra="forbid", frozen=True) + + +class Filters(_Frozen): + """What the host filters at the source, per dimension. `country` takes + ISO 3166 alpha-2 codes, `place` place names, `remote` a bool, `query` + words. The runner never hands a reader a value on a dimension it + declared `none`.""" + + country: Filter = "none" + place: Filter = "none" + remote: Filter = "none" + query: Filter = "none" + + def declared(self) -> dict[str, Filter]: + """The dimensions filtered at the source, `name -> reach`.""" + return {name: reach for name, reach in self.model_dump().items() if reach != "none"} + + +class Fields(_Frozen): + """Which fields the rows state, and how far they can be trusted. + + `pay`: `figures` (`salary_min`, `salary_max`, `currency`), `line` (only + `salary_text`), `none`. `posted`: `trusted` (the host's date is the + posting date), `listed` (a date, but one the host moves, e.g. on an + edit), `none`. The rest: `stated` when the host states it, `none` when + the app has to infer it. + """ + + pay: Literal["figures", "line", "none"] = "none" + posted: Literal["trusted", "listed", "none"] = "none" + remote: Stated = "none" + country: Stated = "none" + seniority: Stated = "none" + employment: Stated = "none" + + +class Key(_Frozen): + """An API key the reader needs: the config entry `sources.search.` + with `fields`, each falling back to the environment variable at the same + position in `env`. The runner resolves it and hands the reader the + values in `Ask.params["key"]`; a reader never reads a key itself.""" + + name: str + fields: tuple[str, ...] + env: tuple[str, ...] = () + + +class Reads(_Frozen): + """What a reader reads and how. One per module, as `READS`. + + `scope`: `board` (one company's whole list) or `search` (many companies, + never a whole list). `listing`: `whole` (one answer) or `paged`. + `page_size` and `ceiling` (paged only): roles a page, and the offset + where the host stops answering whatever is asked (Workday 2,000). + `text`: `listing` (full text in the list), `excerpt` (a teaser in the + list, nothing more), `detail` (one request a role), `none`. `teaser`: + with `text="detail"`, the list also carries a teaser, handed over as + `Row.teaser`. `complete`: `stated` (a total), `end` (an empty page), + `whole` (the one answer), `never` (a search). `places`: `one` (the host + states one), `many` (a list per role), `copies` (the host lists one + posting per place, each with its own id). `robots`: the robots.txt + decision, with the date, taken when the reader was built. `empty`: + `suspect` when an unknown slug answers an empty list rather than an + error. `title`: `provisional` when the listing may not state the title + (a sitemap), which detail then fills. + """ + + scope: Literal["board", "search"] + listing: Literal["whole", "paged"] + page_size: int | None = None + ceiling: int | None = None + text: Literal["listing", "excerpt", "detail", "none"] + teaser: bool = False + complete: Literal["stated", "end", "whole", "never"] + filters: Filters = Filters() + places: Literal["one", "many", "copies"] + fields: Fields = Fields() + robots: str + empty: Literal["trusted", "suspect"] = "trusted" + key: Key | None = None + title: Literal["final", "provisional"] = "final" + + +class Ask(_Frozen): + """What the runner wants from one series of requests. Only the + dimensions the reader declared are set; for a `one` filter each series + carries one value. With `ids`, `countries` and `places` hold the host's + ids from `facets`, not codes or names. `params` are a search's saved + params (and a resolved key under `"key"`). An empty `Ask()` is an + unfiltered read.""" + + countries: tuple[str, ...] = () + places: tuple[str, ...] = () + remote: bool | None = None + query: str = "" + params: dict[str, Any] = {} + + +class Place(TypedDict): + """One place of a role, as the host states it: `text` the host's own + words ("Berlin, Germany"), `remote` True for a remote place ("Remote + (DE)"), False for an office, None when the host does not say. Resolving + `text` to a city, region or country is the app's job, not the reader's.""" + + text: str + remote: bool | None + + +class Row(TypedDict, total=False): + """One role as a reader hands it over. + + `source_id` is the host's id, unique within the board or the search; + `url` a web URL (`_shared.is_web_url`); `places` every place the host + states, never multiplied into rows, never cut to one; `teaser` the + short text the list carries. `location` is the host's own one-line + place, kept while storage reads it. `posted` is an aware datetime or + None; `description` plain text; `remote` True, False, a word or None; + pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and + `employment` only where the host states them, as `Fields` declares. Any + other stated key is kept in `raw` by the runner. + """ + + source_id: Required[str] + title: Required[str] + url: Required[str] + places: Required[list[Place]] + teaser: str + location: str + posted: Any + department: str + description: str + remote: bool | str | None + salary_text: str + salary_min: int | None + salary_max: int | None + currency: str | None + country: str | None + seniority: str | None + employment: str | None + + +#: What a declared module's functions take, positionally. +SIGNATURES: dict[str, tuple[str, ...]] = { + "feed_url": ("slug", "ask"), + "parse": ("body", "slug"), + "page": ("get", "slug", "cursor", "ask"), + "rows": ("decoded", "slug"), + "detail": ("get", "slug", "row"), + "facets": ("get", "slug"), +} + + +def _takes(function: Callable[..., Any], count: int) -> bool: + """Whether `function` accepts `count` positional arguments.""" + try: + signature = inspect.signature(function) + except (TypeError, ValueError): + return True + try: + signature.bind(*([None] * count)) + except TypeError: + return False + return True + + +def problems(module: ModuleType) -> list[str]: + """What `module`'s `READS` gets wrong, on its own or against the + module's functions; empty for a module that declares none.""" + if not hasattr(module, "READS"): + return [] + reads = module.READS + if not isinstance(reads, Reads): + return ["READS is not a Reads"] + has = lambda name: callable(getattr(module, name, None)) # noqa: E731 + out: list[str] = [] + + if reads.listing == "paged" and not (has("page") and has("rows")): + out.append('listing="paged" without page and rows') + if reads.listing == "whole" and not (has("feed_url") and has("parse")): + out.append('listing="whole" without feed_url and parse') + if reads.listing == "whole" and has("page"): + out.append('listing="whole" with page') + if reads.listing == "whole" and (reads.page_size is not None or reads.ceiling is not None): + out.append('page_size and ceiling are for listing="paged"') + if reads.page_size is not None and reads.page_size < 1: + out.append("page_size is not positive") + if reads.ceiling is not None and reads.ceiling < (reads.page_size or 1): + out.append("ceiling is below page_size") + if has("fallback_url") and reads.listing != "whole": + out.append('fallback_url without listing="whole"') + + if reads.text == "detail" and not has("detail"): + out.append('text="detail" without detail') + if has("detail") and reads.text != "detail": + out.append(f'detail with text="{reads.text}"') + if reads.title == "provisional" and not has("detail"): + out.append('title="provisional" without detail to fill it') + if reads.teaser and reads.text != "detail": + out.append('teaser is for text="detail"; the list of "excerpt" is the teaser') + + if (reads.complete == "whole") != (reads.listing == "whole"): + out.append('complete="whole" goes with listing="whole" and only with it') + if reads.scope == "search" and reads.complete not in ("never", "stated"): + out.append('scope="search" needs complete="never" or "stated"') + if reads.scope == "board" and reads.complete == "never": + out.append('scope="board" cannot be complete="never"') + + ids = [name for name, reach in reads.filters.declared().items() if reach == "ids"] + if ids and not has("facets"): + out.append(f"an ids filter ({', '.join(ids)}) without facets") + if has("facets") and not ids: + out.append("facets without an ids filter") + if reads.filters.query == "ids": + out.append('query cannot be "ids"') + + if key := reads.key: + if not _CONFIG_NAME.fullmatch(key.name): + out.append(f"key names no config entry ({key.name!r})") + if not key.fields: + out.append("key has no fields") + if key.env and len(key.env) != len(key.fields): + out.append("key.env does not match key.fields") + if any(not _ENV_NAME.fullmatch(name) for name in key.env): + out.append("key.env has a name that is not an environment variable") + + if not reads.robots.strip(): + out.append("robots is empty: record the robots.txt decision and its date") + + for name, params in SIGNATURES.items(): + if has(name) and not _takes(getattr(module, name), len(params)): + out.append(f"{name} does not take ({', '.join(params)})") + return out diff --git a/plugins/readers/_shared.py b/plugins/readers/_shared.py index b668c27..f76accc 100644 --- a/plugins/readers/_shared.py +++ b/plugins/readers/_shared.py @@ -16,7 +16,9 @@ from typing import Any, Protocol import httpx -#: What every parser turns a feed into: one dict per row, with the keys +#: What an undeclared module's parser turns a feed into; a module that +#: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`. +#: One dict per row, with the keys #: `source_id`, `title`, `location`, `url`, `posted`, `department`, #: `description`, `remote` and `salary_text`, and, where a host states pay, #: `salary_min`, `salary_max` and `currency`. diff --git a/pyproject.toml b/pyproject.toml index 08b71b6..5ecbbf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ ignore = [ "app/models.py" = ["RUF012"] "desk/models.py" = ["RUF012"] "desk/sources/spec.py" = ["RUF012"] +"plugins/readers/_contract.py" = ["RUF012"] # The boilerplate's own test, adapted here but kept in its style. "tests/test_coupling.py" = ["SIM102"] diff --git a/tests/test_readers.py b/tests/test_readers.py index 3739b7f..fe1ccd8 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -33,10 +33,11 @@ FIXTURES = ROOT / "tests" / "fixtures" / "ats" def test_the_registry_finds_every_module() -> None: names = sorted(p.stem for p in (ROOT / "plugins" / "readers").glob("*.py") if not p.stem.startswith("_")) - assert sorted(readers.HOSTS) == names + assert sorted(readers.READERS) == names + assert set(readers.HOSTS) | set(readers.PROVIDERS) == set(names) assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"} - assert all(not readers.problems(module, name) for name, module in readers.HOSTS.items()) + assert all(not readers.problems(module, name) for name, module in readers.READERS.items()) # The Sources page's select and the plugin's description follow the modules. assert {name: module.LABEL for name, module in readers.HOSTS.items()} == ats.HOSTS assert "Teamtailor" in ats.ATSSource.description diff --git a/tests/test_readers_contract.py b/tests/test_readers_contract.py new file mode 100644 index 0000000..da2c8ee --- /dev/null +++ b/tests/test_readers_contract.py @@ -0,0 +1,387 @@ +"""The reader contract, held for every module of `plugins/readers/`. + +Per module: it declares `READS`, the declaration matches its functions, and +its rows keep what the contract says they keep. No network: fixtures stand +in for the host. + +`UNDECLARED` lists the modules that do not declare `READS` yet; their tests +are marked xfail (not strict), and the planner removes a name once the +module declares. A module not in the list must pass. + +Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/` +for searches, named after the module: + +- `-places.`: one listing answer with at least one role at two + or more places. For `listing="whole"` it is the body `parse` gets; for + `listing="paged"` it is what every request of the first `page` call + returns, and `rows` reads that page. Required by the test of a module + that declares `places="many"`, which is skipped until the fixture exists. +- `-places-detail.`: optional, for a host whose places come from + the role's own page: what `detail`'s requests return; its `places` replace + the row's. +- `-teaser.`: one listing answer with a teaser, read the same + way. Required by the test of a module with `text="excerpt"` or + `teaser=True`, skipped until it exists. +- `.slug`: optional, the slug the fixtures are read with, one line; + `acme` without it. +""" + +from __future__ import annotations + +import sys +import types +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pydantic +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import readers # noqa: E402 +from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402 +from plugins.readers import _contract # noqa: E402 + +FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" / "search") + +#: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and +#: C (the search providers, which arrive declared) land. +UNDECLARED = frozenset({ + "ashby", "bamboohr", "deel", "eightfold", "gem", "greenhouse", "hibob", "hrmanager", + "jibe", "jobylon", "lever", + "oracle", "personio", "phenom", "pinpoint", "recruitee", "rippling", "smartrecruiters", + "successfactors", "teamtailor", "workable", "workday", +}) + + +def _params() -> list[Any]: + return [pytest.param(name, marks=pytest.mark.xfail(reason="no READS yet", strict=False)) + if name in UNDECLARED else name for name in readers.READERS] + + +def _declared(name: str) -> tuple[types.ModuleType, Reads]: + module = readers.READERS[name] + reads = getattr(module, "READS", None) + assert isinstance(reads, Reads), f"{name} declares no READS" + return module, reads + + +def _fixture(name: str, kind: str) -> Path | None: + for folder in FIXTURE_DIRS: + found = sorted(folder.glob(f"{name}-{kind}.*")) + if found: + return found[0] + return None + + +def _slug(name: str) -> str: + for folder in FIXTURE_DIRS: + if (path := folder / f"{name}.slug").exists(): + return path.read_text(encoding="utf-8").strip() + return "acme" + + +def _answer(body: bytes) -> Callable[..., bytes]: + return lambda url, **kwargs: body + + +def read_fixture(module: types.ModuleType, body: bytes, slug: str, + detail_body: bytes | None = None) -> list[dict[str, Any]]: + """The rows `module` makes of one listing answer, as the runner reads + them: `parse` for a whole listing, the first `page` and its `rows` for a + paged one, and, given `detail_body`, each row's `places` from `detail`.""" + reads: Reads = module.READS + if reads.listing == "whole": + rows = list(module.parse(body, slug)) + else: + decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, Ask()) + rows = list(module.rows(decoded, slug)) + if detail_body is not None: + for row in rows: + found = module.detail(_answer(detail_body), slug, row) + if "places" in found: + row["places"] = found["places"] + return rows + + +def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]: + """What rows get wrong about `places` under the declared trait.""" + out = [] + for row in rows: + found = row.get("places") + rid = row.get("source_id") + if not isinstance(found, list): + out.append(f"{rid}: places is not a list") + continue + for place in found: + if not (isinstance(place, dict) and set(place) == {"text", "remote"} + and isinstance(place["text"], str) and place["text"].strip() + and place["remote"] in (True, False, None)): + out.append(f"{rid}: {place!r} is not a Place") + if places in ("one", "copies") and len(found) > 1: + out.append(f'{rid}: {len(found)} places under places="{places}"') + if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows): + out.append('no row has two places: a reader with places="many" keeps every place') + return out + + +# ------------------------------------------------------------ every module + +def test_the_undeclared_list_names_only_readers() -> None: + assert set(readers.READERS) >= UNDECLARED + + +def test_undeclared_lists_the_modules_without_reads() -> None: + assert set(readers.undeclared()) == {name for name, module in readers.READERS.items() + if not hasattr(module, "READS")} + + +@pytest.mark.parametrize("name", _params()) +def test_a_reader_declares_reads(name: str) -> None: + _declared(name) + + +@pytest.mark.parametrize("name", _params()) +def test_a_declaration_matches_the_functions(name: str) -> None: + module, _reads = _declared(name) + assert readers.problems(module, name) == [] + expected = "search" if module.READS.scope == "search" else "board" + assert name in (readers.PROVIDERS if expected == "search" else readers.HOSTS) + + +@pytest.mark.parametrize("name", _params()) +def test_a_reader_keeps_every_place(name: str) -> None: + module, reads = _declared(name) + fixture = _fixture(name, "places") + if fixture is None: + if reads.places == "many": + pytest.skip(f"no fixture {name}-places.* yet") + pytest.skip(f'places="{reads.places}" and no fixture') + detail = _fixture(name, "places-detail") + rows = read_fixture(module, fixture.read_bytes(), _slug(name), + detail.read_bytes() if detail else None) + assert rows, f"{fixture.name} gave no rows" + assert place_problems(rows, reads.places) == [] + + +@pytest.mark.parametrize("name", _params()) +def test_a_reader_keeps_the_teaser(name: str) -> None: + module, reads = _declared(name) + if not (reads.text == "excerpt" or reads.teaser): + pytest.skip("the list carries no teaser") + fixture = _fixture(name, "teaser") + if fixture is None: + pytest.skip(f"no fixture {name}-teaser.* yet") + rows = read_fixture(module, fixture.read_bytes(), _slug(name)) + assert any(isinstance(row.get("teaser"), str) and row["teaser"].strip() for row in rows) + + +# ------------------------------------------------------------ the checks + +def _reader(**functions: Any) -> Any: + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", (r"fake\.example/([a-z]+)",) + module.slug_ok = lambda slug: True + module.allowed = lambda slug: lambda host: host == "fake.example" + module.careers_url = lambda slug: "https://fake.example/" + slug + for name, function in functions.items(): + setattr(module, name, function) + return module + + +def _whole(**functions: Any) -> Any: + return _reader(**{"feed_url": lambda slug, ask: "https://fake.example/" + slug, + "parse": lambda body, slug: []} | functions) + + +def _paged(**functions: Any) -> Any: + return _reader(**{"page": lambda get, slug, cursor, ask: (None, [], None, None), + "rows": lambda decoded, slug: []} | functions) + + +_ROBOTS = "read: no robots.txt rule on /api/ (checked 2026-09-30)" + + +def _reads(**traits: Any) -> Reads: + base: dict[str, Any] = {"scope": "board", "listing": "whole", "text": "listing", + "complete": "whole", "places": "one", "robots": _ROBOTS} + return Reads(**(base | traits)) + + +def test_a_good_declaration_has_no_problems() -> None: + module = _whole() + module.READS = _reads() + assert readers.problems(module, "fake") == [] + module = _paged(detail=lambda get, slug, row: {}, + facets=lambda get, slug: {"country": {"DE": "1"}}) + module.READS = _reads(listing="paged", page_size=20, ceiling=2000, text="detail", + teaser=True, complete="stated", filters=Filters(country="ids"), + places="many", fields=Fields(posted="listed", remote="stated"), + empty="suspect", title="provisional") + assert readers.problems(module, "fake") == [] + + +def test_a_module_without_reads_is_checked_as_before() -> None: + module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: []) + assert readers.problems(module, "fake") == [] + assert readers.scope(module) == "board" + assert readers.undeclared({"fake": module}) == ["fake"] + + +@pytest.mark.parametrize(("module", "reads", "problem"), [ + (_reader(feed_url=lambda slug, ask: "", parse=lambda b, s: []), + {"listing": "paged", "complete": "stated"}, 'listing="paged" without page and rows'), + (_paged(), {}, 'listing="whole" without feed_url and parse'), + (_whole(), {"complete": "stated"}, 'complete="whole" goes with listing="whole"'), + (_whole(), {"page_size": 20}, 'page_size and ceiling are for listing="paged"'), + (_paged(), {"listing": "paged", "complete": "end", "page_size": 100, "ceiling": 50}, + "ceiling is below page_size"), + (_paged(fallback_url=lambda slug: ""), {"listing": "paged", "complete": "end"}, + 'fallback_url without listing="whole"'), + (_whole(), {"text": "detail"}, 'text="detail" without detail'), + (_whole(detail=lambda g, s, r: {}), {}, 'detail with text="listing"'), + (_whole(), {"teaser": True, "text": "excerpt"}, "teaser is for text=\"detail\""), + (_whole(), {"title": "provisional"}, 'title="provisional" without detail'), + (_whole(), {"filters": Filters(country="ids")}, "an ids filter (country) without facets"), + (_whole(facets=lambda g, s: {}), {"filters": Filters(country="one")}, + "facets without an ids filter"), + (_whole(facets=lambda g, s: {}), {"filters": Filters(query="ids")}, + 'query cannot be "ids"'), + (_paged(), {"scope": "search", "listing": "paged", "complete": "end"}, + 'scope="search" needs complete="never" or "stated"'), + (_paged(), {"listing": "paged", "complete": "never"}, + 'scope="board" cannot be complete="never"'), + (_whole(), {"key": Key(name="Adzuna Key", fields=("app_id",))}, + "key names no config entry"), + (_whole(), {"key": Key(name="adzuna", fields=())}, "key has no fields"), + (_whole(), {"key": Key(name="adzuna", fields=("app_id", "app_key"), + env=("ADZUNA_APP_ID",))}, + "key.env does not match key.fields"), + (_whole(), {"robots": " "}, "robots is empty"), + (_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {}, + "feed_url does not take (slug, ask)"), + (_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []), + {"listing": "paged", "complete": "end"}, "page does not take (get, slug, cursor, ask)"), +]) +def test_the_declaration_check_names_each_problem( + module: Any, reads: dict[str, Any], problem: str) -> None: + module.READS = _reads(**reads) + found = readers.problems(module, "fake") + assert any(p.startswith(problem) for p in found), found + + +def test_reads_that_is_not_a_reads_is_refused() -> None: + module = _whole() + module.READS = {"scope": "board"} + assert "READS is not a Reads" in readers.problems(module, "fake") + + +def test_a_search_needs_fewer_names_than_a_board() -> None: + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL = "fake", "Fake" + module.allowed = lambda slug: lambda host: True + module.page = lambda get, slug, cursor, ask: (None, [], 0, None) + module.rows = lambda decoded, slug: [] + module.READS = _reads(scope="search", listing="paged", complete="never", + filters=Filters(query="one"), + key=Key(name="adzuna", fields=("app_id", "app_key"), + env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY"))) + assert readers.problems(module, "fake") == [] + assert readers.scope(module) == "search" + del module.READS + assert "no careers_url" in readers.problems(module, "fake") + + +def test_a_misspelt_trait_fails_where_it_is_declared() -> None: + with pytest.raises(pydantic.ValidationError): + _reads(listing="pages") + with pytest.raises(pydantic.ValidationError): + Filters(country="some") + with pytest.raises(pydantic.ValidationError): + _reads(robots=None) + with pytest.raises(pydantic.ValidationError): + Reads(scope="board", listing="whole", text="listing", complete="whole", + places="one") # type: ignore[call-arg] # robots is required + with pytest.raises(pydantic.ValidationError): + _reads(colour="blue") # extra keys are refused + + +def test_a_declaration_is_frozen() -> None: + reads = _reads() + with pytest.raises(pydantic.ValidationError): + reads.places = "many" # type: ignore[misc] + + +def test_the_loader_refuses_a_module_whose_declaration_breaks_the_contract( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + package = tmp_path / "readersbad" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "odd.py").write_text( + "from plugins.readers import Reads\n" + "NAME = 'odd'\nLABEL = 'Odd'\nPATTERNS = ()\n" + "def slug_ok(slug): return True\n" + "def allowed(slug): return lambda h: True\n" + "def careers_url(slug): return ''\n" + "def feed_url(slug, ask): return ''\n" + "def parse(body, slug): return []\n" + "READS = Reads(scope='board', listing='whole', text='detail', complete='whole',\n" + " places='many', robots='read (checked 2026-09-30)')\n") + monkeypatch.syspath_prepend(str(tmp_path)) + with pytest.raises(ImportError, match=r'readersbad\.odd .*text="detail" without detail'): + readers.load([str(package)], "readersbad") + + +def test_filters_list_what_is_filtered() -> None: + assert Filters(country="one", query="many").declared() == {"country": "one", + "query": "many"} + + +# ------------------------------------------------------------ the fixture check + +def _fake_many(rows: list[dict[str, Any]]) -> Any: + module = _reader(feed_url=lambda slug, ask: "", parse=lambda body, slug: rows) + module.READS = _reads(places="many") + return module + + +def test_the_place_check_passes_a_role_at_two_places() -> None: + rows = [{"source_id": "1", "places": [{"text": "Berlin", "remote": False}, + {"text": "Remote (DE)", "remote": True}]}] + assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == [] + + +def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None: + joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}] + assert place_problems(joined, "many") == [ + 'no row has two places: a reader with places="many" keeps every place'] + assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [ + "2: places is not a list"] + assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [ + "3: ('Berlin', False) is not a Place"] + + +def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None: + seen: list[bytes] = [] + + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + seen.append(get("https://fake.example/list")) + return ([{"id": "1"}], [{"id": "1"}], 1, None) + + module = _paged(page=page, rows=lambda decoded, slug: [{"source_id": d["id"], "places": []} + for d in decoded], + detail=lambda get, slug, row: {"places": [ + {"text": get("https://fake.example/1").decode(), "remote": None}, + {"text": "Munich", "remote": None}]}) + module.READS = _reads(listing="paged", complete="stated", text="detail", places="many") + rows = read_fixture(module, b"Berlin", "acme", detail_body=b"Berlin") + assert seen == [b"Berlin"] + assert place_problems(rows, "many") == [] + + +def test_the_contract_types_are_what_the_package_exports() -> None: + assert readers.Reads is _contract.Reads and readers.Row is _contract.Row + assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}