Files
jobboard/plugins/readers/_contract.py
T
WolfandClaude Fable 5.1 2349fc424b Require READS, search params and a dated robots record; record robots.txt per requested path
Every reader declares, so undeclared(), the scope() fallback, robots_page and
ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may
carry company, places gains none. Each robots record names the paths the
reader requests, from robots.txt read again on 2026-09-30. Platsbanken states
remote, HR Manager states no place.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-30 15:51:13 +02:00

333 lines
14 KiB
Python

"""The types of the reader contract, and the check of a module's declaration
against its functions. The prose is the package's docstring; this file is
what it names.
A reader declares `READS = Reads(...)`: what it reads (a board or a search),
how the listing arrives, where the text is, how completeness is known, what
it can filter at the source, how it states places and which fields it
states. The runner reads the declaration and does the rest. Values are
checked by pydantic where the module is declared, so a misspelt trait fails
the import; what depends on the module's functions, or on two traits at
once, is `problems()`, which the package's loader runs and turns into an
ImportError naming every problem.
Signatures of a module that declares `READS`, with `Get` and `Page` from
`._shared`:
def feed_url(slug: str, ask: Ask) -> str # listing="whole"
def parse(body: bytes, slug: str) -> list[Row] # listing="whole"
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged"
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
def clean_params(params: dict[str, Any]) -> dict[str, Any] # scope="search"
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
`country`, a place name for `place`, any word for `remote`. A dimension the
host has no facet for is left out, and the runner then reads it unfiltered;
a dimension present without a wanted value means the host has none of it.
A module without `READS` fails the import: every reader declares.
"""
from __future__ import annotations
import inspect
import re
from collections.abc import Callable
from types import ModuleType
from typing import Any, Literal, Required, TypedDict
from pydantic import ConfigDict
from desk.models import Model
#: A filter dimension's reach at the source: `none`, not filtered; `one`, one
#: value a request (the runner makes one series per value and unions them by
#: id); `many`, a list in one request; `ids`, values the host names by its own
#: ids, which `facets` looks up.
Filter = Literal["none", "one", "many", "ids"]
#: Whether the reader states a field: `stated` from the host, `none` never.
Stated = Literal["stated", "none"]
#: A config entry name: `sources.search.<name>` in `config.yaml`.
_CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
#: An environment variable name.
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
#: The date a robots.txt record must carry: when the file was read.
_DATE = re.compile(r"20\d\d-\d\d-\d\d")
class _Frozen(Model):
"""A declaration: immutable, extra keys refused."""
model_config = ConfigDict(from_attributes=True, extra="forbid", frozen=True)
class Filters(_Frozen):
"""What the host filters at the source, per dimension. `country` takes
ISO 3166 alpha-2 codes, `place` place names, `remote` a bool, `query`
words. The runner never hands a reader a value on a dimension it
declared `none`. A board host with `query="one"` also takes the agents'
fetch terms, one term a series (`Ask.terms`, see `_runner.ask_for`)."""
country: Filter = "none"
place: Filter = "none"
remote: Filter = "none"
query: Filter = "none"
def declared(self) -> dict[str, Filter]:
"""The dimensions filtered at the source, `name -> reach`."""
return {name: reach for name, reach in self.model_dump().items() if reach != "none"}
class Fields(_Frozen):
"""Which fields the rows state, and how far they can be trusted.
`pay`: `figures` (`salary_min`, `salary_max`, `currency`), `line` (only
`salary_text`), `none`. `posted`: `trusted` (the host's date is the
posting date), `listed` (a date, but one the host moves, e.g. on an
edit), `none`. The rest: `stated` when the host states it, `none` when
the app has to infer it.
"""
pay: Literal["figures", "line", "none"] = "none"
posted: Literal["trusted", "listed", "none"] = "none"
remote: Stated = "none"
country: Stated = "none"
seniority: Stated = "none"
employment: Stated = "none"
class Key(_Frozen):
"""An API key the reader needs: the config entry `sources.search.<name>`
with `fields`, each falling back to the environment variable at the same
position in `env`. The runner resolves it and hands the reader the
values in `Ask.params["key"]`; a reader never reads a key itself."""
name: str
fields: tuple[str, ...]
env: tuple[str, ...] = ()
class Reads(_Frozen):
"""What a reader reads and how. One per module, as `READS`.
`scope`: `board` (one company's whole list) or `search` (many companies,
never a whole list). `listing`: `whole` (one answer) or `paged`.
`page_size` and `ceiling` (paged only): roles a page, and the offset
where the host stops answering whatever is asked (Workday 2,000).
`text`: `listing` (full text in the list), `excerpt` (a teaser in the
list, nothing more), `detail` (one request a role), `none`. `teaser`:
with `text="detail"`, the list also carries a teaser, handed over as
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
`whole` (the one answer), `never` (a search). `places`: `one` (the host
states one), `many` (a list per role), `copies` (the host lists one
posting per place, each with its own id), `none` (the host states no
place; every row has `places == []`). `robots`: the robots.txt
decision for the paths the reader requests, with the date the file was
read. `empty`: `suspect` when an unknown slug answers an empty list
rather than an error. `title`: `provisional` when the listing may not
state the title (a sitemap), which detail then fills; nothing reads it
yet, since `_runner.DETAIL_FIELDS` has no `title`.
"""
scope: Literal["board", "search"]
listing: Literal["whole", "paged"]
page_size: int | None = None
ceiling: int | None = None
text: Literal["listing", "excerpt", "detail", "none"]
teaser: bool = False
complete: Literal["stated", "end", "whole", "never"]
filters: Filters = Filters()
places: Literal["one", "many", "copies", "none"]
fields: Fields = Fields()
robots: str
empty: Literal["trusted", "suspect"] = "trusted"
key: Key | None = None
title: Literal["final", "provisional"] = "final"
class Ask(_Frozen):
"""What the runner wants from one series of requests. Only the
dimensions the reader declared are set; for a `one` filter each series
carries one value. With `ids`, `countries` and `places` hold the host's
ids from `facets`, not codes or names. `params` are a search's saved
params (and a resolved key under `"key"`). An empty `Ask()` is an
unfiltered read.
`remote=True` asks for remote roles. A reach of countries and remote
means "in these countries, or remote", which no host can ask in one
request (two facets intersect), so the runner makes the remote roles a
series of their own: a reader never gets both `remote=True` and a
country or a place. With `remote="ids"`, `remote_ids` holds the host's
ids of its remote values (`facets(...)["remote"]`, any key), and
`remote` stays True.
`terms` are the agents' fetch terms a board is narrowed by (see
`app/jobs/reach.py`). The runner makes one series per term, each with
that term as `query`, so a reader only ever reads `query`. A read with
terms is never the whole board: it closes nothing."""
countries: tuple[str, ...] = ()
places: tuple[str, ...] = ()
remote: bool | None = None
remote_ids: tuple[str, ...] = ()
query: str = ""
terms: tuple[str, ...] = ()
params: dict[str, Any] = {}
class Place(TypedDict):
"""One place of a role, as the host states it: `text` the host's own
words ("Berlin, Germany"), `remote` True for a remote place ("Remote
(DE)"), False for an office, None when the host does not say. Resolving
`text` to a city, region or country is the app's job, not the reader's."""
text: str
remote: bool | None
class Row(TypedDict, total=False):
"""One role as a reader hands it over.
`source_id` is the host's id, unique within the board or the search;
`url` a web URL (`_shared.is_web_url`); `places` every place the host
states, never multiplied into rows, never cut to one; `teaser` the
short text the list carries. `location` is the host's own one-line
place, kept while storage reads it. `posted` is an aware datetime or
None; `description` plain text; `remote` True, False, a word or None;
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
`employment` only where the host states them, as `Fields` declares.
`company` is the employer's name on a search row, which lists many
(`plugins/search.py` stores it); a board row carries none, the board is
the company. Any other stated key is kept in `raw` by the runner.
"""
source_id: Required[str]
title: Required[str]
url: Required[str]
places: Required[list[Place]]
teaser: str
location: str
posted: Any
department: str
description: str
remote: bool | str | None
salary_text: str
salary_min: int | None
salary_max: int | None
currency: str | None
country: str | None
seniority: str | None
employment: str | None
company: str
#: What a declared module's functions take, positionally.
SIGNATURES: dict[str, tuple[str, ...]] = {
"feed_url": ("slug", "ask"),
"parse": ("body", "slug"),
"page": ("get", "slug", "cursor", "ask"),
"rows": ("decoded", "slug"),
"detail": ("get", "slug", "row"),
"facets": ("get", "slug"),
"clean_params": ("params",),
}
def _takes(function: Callable[..., Any], count: int) -> bool:
"""Whether `function` accepts `count` positional arguments."""
try:
signature = inspect.signature(function)
except (TypeError, ValueError):
return True
try:
signature.bind(*([None] * count))
except TypeError:
return False
return True
def problems(module: ModuleType) -> list[str]:
"""What `module`'s `READS` gets wrong, on its own or against the
module's functions; "no READS" for a module that declares none."""
if not hasattr(module, "READS"):
return ["no READS: every reader declares what it reads"]
reads = module.READS
if not isinstance(reads, Reads):
return ["READS is not a Reads"]
has = lambda name: callable(getattr(module, name, None)) # noqa: E731
out: list[str] = []
if reads.listing == "paged" and not (has("page") and has("rows")):
out.append('listing="paged" without page and rows')
if reads.listing == "whole" and not (has("feed_url") and has("parse")):
out.append('listing="whole" without feed_url and parse')
if reads.listing == "whole" and has("page"):
out.append('listing="whole" with page')
if reads.listing == "whole" and (reads.page_size is not None or reads.ceiling is not None):
out.append('page_size and ceiling are for listing="paged"')
if reads.page_size is not None and reads.page_size < 1:
out.append("page_size is not positive")
if reads.ceiling is not None and reads.ceiling < (reads.page_size or 1):
out.append("ceiling is below page_size")
if has("fallback_url") and reads.listing != "whole":
out.append('fallback_url without listing="whole"')
if reads.text == "detail" and not has("detail"):
out.append('text="detail" without detail')
if has("detail") and reads.text != "detail":
out.append(f'detail with text="{reads.text}"')
if reads.title == "provisional" and not has("detail"):
out.append('title="provisional" without detail to fill it')
if reads.teaser and reads.text != "detail":
out.append('teaser is for text="detail"; the list of "excerpt" is the teaser')
if (reads.complete == "whole") != (reads.listing == "whole"):
out.append('complete="whole" goes with listing="whole" and only with it')
if reads.scope == "search" and reads.complete not in ("never", "stated"):
out.append('scope="search" needs complete="never" or "stated"')
if reads.scope == "board" and reads.complete == "never":
out.append('scope="board" cannot be complete="never"')
ids = [name for name, reach in reads.filters.declared().items() if reach == "ids"]
if ids and not has("facets"):
out.append(f"an ids filter ({', '.join(ids)}) without facets")
if has("facets") and not ids:
out.append("facets without an ids filter")
if reads.filters.query == "ids":
out.append('query cannot be "ids"')
if key := reads.key:
if not _CONFIG_NAME.fullmatch(key.name):
out.append(f"key names no config entry ({key.name!r})")
if not key.fields:
out.append("key has no fields")
if key.env and len(key.env) != len(key.fields):
out.append("key.env does not match key.fields")
if any(not _ENV_NAME.fullmatch(name) for name in key.env):
out.append("key.env has a name that is not an environment variable")
if not reads.robots.strip():
out.append("robots is empty: record the robots.txt decision and its date")
elif not _DATE.search(reads.robots):
out.append("robots has no date: record when robots.txt was read (YYYY-MM-DD)")
if reads.scope == "search" and hasattr(module, "PARAMS"):
params = module.PARAMS
if not isinstance(params, dict) or not params:
out.append("PARAMS is not a dict of saved param -> filter dimension")
else:
dimensions = set(Filters.model_fields)
out += [f"PARAMS maps {param!r} to {dimension!r}, not a filter dimension"
for param, dimension in params.items() if dimension not in dimensions]
for name, params in SIGNATURES.items():
if has(name) and not _takes(getattr(module, name), len(params)):
out.append(f"{name} does not take ({', '.join(params)})")
return out