"""The types of the reader contract, and the check of a module's declaration against its functions. The prose is the package's docstring; this file is what it names. A reader declares `READS = Reads(...)`: what it reads (a board or a search), how the listing arrives, where the text is, how completeness is known, what it can filter at the source, how it states places and which fields it states. The runner reads the declaration and does the rest. Values are checked by pydantic where the module is declared, so a misspelt trait fails the import; what depends on the module's functions, or on two traits at once, is `problems()`, which the package's loader runs and turns into an ImportError naming every problem. Signatures of a module that declares `READS`, with `Get` and `Page` from `._shared`: def feed_url(slug: str, ask: Ask) -> str # listing="whole" def parse(body: bytes, slug: str) -> list[Row] # listing="whole" def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged" def rows(decoded: Any, slug: str) -> list[Row] # listing="paged" def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail" def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter A module without `READS` keeps the signatures without `ask` until it declares; `undeclared()` in the package lists those modules. """ from __future__ import annotations import inspect import re from collections.abc import Callable from types import ModuleType from typing import Any, Literal, Required, TypedDict from pydantic import ConfigDict from desk.models import Model #: A filter dimension's reach at the source: `none`, not filtered; `one`, one #: value a request (the runner makes one series per value and unions them by #: id); `many`, a list in one request; `ids`, values the host names by its own #: ids, which `facets` looks up. Filter = Literal["none", "one", "many", "ids"] #: Whether the reader states a field: `stated` from the host, `none` never. Stated = Literal["stated", "none"] #: A config entry name: `sources.search.` in `config.yaml`. _CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*") #: An environment variable name. _ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*") class _Frozen(Model): """A declaration: immutable, extra keys refused.""" model_config = ConfigDict(from_attributes=True, extra="forbid", frozen=True) class Filters(_Frozen): """What the host filters at the source, per dimension. `country` takes ISO 3166 alpha-2 codes, `place` place names, `remote` a bool, `query` words. The runner never hands a reader a value on a dimension it declared `none`.""" country: Filter = "none" place: Filter = "none" remote: Filter = "none" query: Filter = "none" def declared(self) -> dict[str, Filter]: """The dimensions filtered at the source, `name -> reach`.""" return {name: reach for name, reach in self.model_dump().items() if reach != "none"} class Fields(_Frozen): """Which fields the rows state, and how far they can be trusted. `pay`: `figures` (`salary_min`, `salary_max`, `currency`), `line` (only `salary_text`), `none`. `posted`: `trusted` (the host's date is the posting date), `listed` (a date, but one the host moves, e.g. on an edit), `none`. The rest: `stated` when the host states it, `none` when the app has to infer it. """ pay: Literal["figures", "line", "none"] = "none" posted: Literal["trusted", "listed", "none"] = "none" remote: Stated = "none" country: Stated = "none" seniority: Stated = "none" employment: Stated = "none" class Key(_Frozen): """An API key the reader needs: the config entry `sources.search.` with `fields`, each falling back to the environment variable at the same position in `env`. The runner resolves it and hands the reader the values in `Ask.params["key"]`; a reader never reads a key itself.""" name: str fields: tuple[str, ...] env: tuple[str, ...] = () class Reads(_Frozen): """What a reader reads and how. One per module, as `READS`. `scope`: `board` (one company's whole list) or `search` (many companies, never a whole list). `listing`: `whole` (one answer) or `paged`. `page_size` and `ceiling` (paged only): roles a page, and the offset where the host stops answering whatever is asked (Workday 2,000). `text`: `listing` (full text in the list), `excerpt` (a teaser in the list, nothing more), `detail` (one request a role), `none`. `teaser`: with `text="detail"`, the list also carries a teaser, handed over as `Row.teaser`. `complete`: `stated` (a total), `end` (an empty page), `whole` (the one answer), `never` (a search). `places`: `one` (the host states one), `many` (a list per role), `copies` (the host lists one posting per place, each with its own id). `robots`: the robots.txt decision, with the date, taken when the reader was built. `empty`: `suspect` when an unknown slug answers an empty list rather than an error. `title`: `provisional` when the listing may not state the title (a sitemap), which detail then fills. """ scope: Literal["board", "search"] listing: Literal["whole", "paged"] page_size: int | None = None ceiling: int | None = None text: Literal["listing", "excerpt", "detail", "none"] teaser: bool = False complete: Literal["stated", "end", "whole", "never"] filters: Filters = Filters() places: Literal["one", "many", "copies"] fields: Fields = Fields() robots: str empty: Literal["trusted", "suspect"] = "trusted" key: Key | None = None title: Literal["final", "provisional"] = "final" class Ask(_Frozen): """What the runner wants from one series of requests. Only the dimensions the reader declared are set; for a `one` filter each series carries one value. With `ids`, `countries` and `places` hold the host's ids from `facets`, not codes or names. `params` are a search's saved params (and a resolved key under `"key"`). An empty `Ask()` is an unfiltered read.""" countries: tuple[str, ...] = () places: tuple[str, ...] = () remote: bool | None = None query: str = "" params: dict[str, Any] = {} class Place(TypedDict): """One place of a role, as the host states it: `text` the host's own words ("Berlin, Germany"), `remote` True for a remote place ("Remote (DE)"), False for an office, None when the host does not say. Resolving `text` to a city, region or country is the app's job, not the reader's.""" text: str remote: bool | None class Row(TypedDict, total=False): """One role as a reader hands it over. `source_id` is the host's id, unique within the board or the search; `url` a web URL (`_shared.is_web_url`); `places` every place the host states, never multiplied into rows, never cut to one; `teaser` the short text the list carries. `location` is the host's own one-line place, kept while storage reads it. `posted` is an aware datetime or None; `description` plain text; `remote` True, False, a word or None; pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and `employment` only where the host states them, as `Fields` declares. Any other stated key is kept in `raw` by the runner. """ source_id: Required[str] title: Required[str] url: Required[str] places: Required[list[Place]] teaser: str location: str posted: Any department: str description: str remote: bool | str | None salary_text: str salary_min: int | None salary_max: int | None currency: str | None country: str | None seniority: str | None employment: str | None #: What a declared module's functions take, positionally. SIGNATURES: dict[str, tuple[str, ...]] = { "feed_url": ("slug", "ask"), "parse": ("body", "slug"), "page": ("get", "slug", "cursor", "ask"), "rows": ("decoded", "slug"), "detail": ("get", "slug", "row"), "facets": ("get", "slug"), } def _takes(function: Callable[..., Any], count: int) -> bool: """Whether `function` accepts `count` positional arguments.""" try: signature = inspect.signature(function) except (TypeError, ValueError): return True try: signature.bind(*([None] * count)) except TypeError: return False return True def problems(module: ModuleType) -> list[str]: """What `module`'s `READS` gets wrong, on its own or against the module's functions; empty for a module that declares none.""" if not hasattr(module, "READS"): return [] reads = module.READS if not isinstance(reads, Reads): return ["READS is not a Reads"] has = lambda name: callable(getattr(module, name, None)) # noqa: E731 out: list[str] = [] if reads.listing == "paged" and not (has("page") and has("rows")): out.append('listing="paged" without page and rows') if reads.listing == "whole" and not (has("feed_url") and has("parse")): out.append('listing="whole" without feed_url and parse') if reads.listing == "whole" and has("page"): out.append('listing="whole" with page') if reads.listing == "whole" and (reads.page_size is not None or reads.ceiling is not None): out.append('page_size and ceiling are for listing="paged"') if reads.page_size is not None and reads.page_size < 1: out.append("page_size is not positive") if reads.ceiling is not None and reads.ceiling < (reads.page_size or 1): out.append("ceiling is below page_size") if has("fallback_url") and reads.listing != "whole": out.append('fallback_url without listing="whole"') if reads.text == "detail" and not has("detail"): out.append('text="detail" without detail') if has("detail") and reads.text != "detail": out.append(f'detail with text="{reads.text}"') if reads.title == "provisional" and not has("detail"): out.append('title="provisional" without detail to fill it') if reads.teaser and reads.text != "detail": out.append('teaser is for text="detail"; the list of "excerpt" is the teaser') if (reads.complete == "whole") != (reads.listing == "whole"): out.append('complete="whole" goes with listing="whole" and only with it') if reads.scope == "search" and reads.complete not in ("never", "stated"): out.append('scope="search" needs complete="never" or "stated"') if reads.scope == "board" and reads.complete == "never": out.append('scope="board" cannot be complete="never"') ids = [name for name, reach in reads.filters.declared().items() if reach == "ids"] if ids and not has("facets"): out.append(f"an ids filter ({', '.join(ids)}) without facets") if has("facets") and not ids: out.append("facets without an ids filter") if reads.filters.query == "ids": out.append('query cannot be "ids"') if key := reads.key: if not _CONFIG_NAME.fullmatch(key.name): out.append(f"key names no config entry ({key.name!r})") if not key.fields: out.append("key has no fields") if key.env and len(key.env) != len(key.fields): out.append("key.env does not match key.fields") if any(not _ENV_NAME.fullmatch(name) for name in key.env): out.append("key.env has a name that is not an environment variable") if not reads.robots.strip(): out.append("robots is empty: record the robots.txt decision and its date") for name, params in SIGNATURES.items(): if has(name) and not _takes(getattr(module, name), len(params)): out.append(f"{name} does not take ({', '.join(params)})") return out