Files
jobboard/plugins/readers/_contract.py
T
WolfandClaude Opus 5.5 3462a4abe7 Read big boards within the agents' reach: countries and remote from the enabled agents, Europe for favourites, filtered at the source by six hosts
app/jobs/reach.py derives each board's reach once per pull and the pull hands it to the runner through the batch payload. SmartRecruiters, Eightfold and SuccessFactors filter one country a request, Phenom a list, Workday and Oracle by host ids from facets. Remote beside countries is a series of its own, since two facets intersect.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 09:55:18 +02:00

306 lines
12 KiB
Python

"""The types of the reader contract, and the check of a module's declaration
against its functions. The prose is the package's docstring; this file is
what it names.
A reader declares `READS = Reads(...)`: what it reads (a board or a search),
how the listing arrives, where the text is, how completeness is known, what
it can filter at the source, how it states places and which fields it
states. The runner reads the declaration and does the rest. Values are
checked by pydantic where the module is declared, so a misspelt trait fails
the import; what depends on the module's functions, or on two traits at
once, is `problems()`, which the package's loader runs and turns into an
ImportError naming every problem.
Signatures of a module that declares `READS`, with `Get` and `Page` from
`._shared`:
def feed_url(slug: str, ask: Ask) -> str # listing="whole"
def parse(body: bytes, slug: str) -> list[Row] # listing="whole"
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged"
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
`country`, a place name for `place`, any word for `remote`. A dimension the
host has no facet for is left out, and the runner then reads it unfiltered;
a dimension present without a wanted value means the host has none of it.
A module without `READS` keeps the signatures without `ask` until it
declares; `undeclared()` in the package lists those modules.
"""
from __future__ import annotations
import inspect
import re
from collections.abc import Callable
from types import ModuleType
from typing import Any, Literal, Required, TypedDict
from pydantic import ConfigDict
from desk.models import Model
#: A filter dimension's reach at the source: `none`, not filtered; `one`, one
#: value a request (the runner makes one series per value and unions them by
#: id); `many`, a list in one request; `ids`, values the host names by its own
#: ids, which `facets` looks up.
Filter = Literal["none", "one", "many", "ids"]
#: Whether the reader states a field: `stated` from the host, `none` never.
Stated = Literal["stated", "none"]
#: A config entry name: `sources.search.<name>` in `config.yaml`.
_CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
#: An environment variable name.
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
class _Frozen(Model):
"""A declaration: immutable, extra keys refused."""
model_config = ConfigDict(from_attributes=True, extra="forbid", frozen=True)
class Filters(_Frozen):
"""What the host filters at the source, per dimension. `country` takes
ISO 3166 alpha-2 codes, `place` place names, `remote` a bool, `query`
words. The runner never hands a reader a value on a dimension it
declared `none`."""
country: Filter = "none"
place: Filter = "none"
remote: Filter = "none"
query: Filter = "none"
def declared(self) -> dict[str, Filter]:
"""The dimensions filtered at the source, `name -> reach`."""
return {name: reach for name, reach in self.model_dump().items() if reach != "none"}
class Fields(_Frozen):
"""Which fields the rows state, and how far they can be trusted.
`pay`: `figures` (`salary_min`, `salary_max`, `currency`), `line` (only
`salary_text`), `none`. `posted`: `trusted` (the host's date is the
posting date), `listed` (a date, but one the host moves, e.g. on an
edit), `none`. The rest: `stated` when the host states it, `none` when
the app has to infer it.
"""
pay: Literal["figures", "line", "none"] = "none"
posted: Literal["trusted", "listed", "none"] = "none"
remote: Stated = "none"
country: Stated = "none"
seniority: Stated = "none"
employment: Stated = "none"
class Key(_Frozen):
"""An API key the reader needs: the config entry `sources.search.<name>`
with `fields`, each falling back to the environment variable at the same
position in `env`. The runner resolves it and hands the reader the
values in `Ask.params["key"]`; a reader never reads a key itself."""
name: str
fields: tuple[str, ...]
env: tuple[str, ...] = ()
class Reads(_Frozen):
"""What a reader reads and how. One per module, as `READS`.
`scope`: `board` (one company's whole list) or `search` (many companies,
never a whole list). `listing`: `whole` (one answer) or `paged`.
`page_size` and `ceiling` (paged only): roles a page, and the offset
where the host stops answering whatever is asked (Workday 2,000).
`text`: `listing` (full text in the list), `excerpt` (a teaser in the
list, nothing more), `detail` (one request a role), `none`. `teaser`:
with `text="detail"`, the list also carries a teaser, handed over as
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
`whole` (the one answer), `never` (a search). `places`: `one` (the host
states one), `many` (a list per role), `copies` (the host lists one
posting per place, each with its own id). `robots`: the robots.txt
decision, with the date, taken when the reader was built. `empty`:
`suspect` when an unknown slug answers an empty list rather than an
error. `title`: `provisional` when the listing may not state the title
(a sitemap), which detail then fills.
"""
scope: Literal["board", "search"]
listing: Literal["whole", "paged"]
page_size: int | None = None
ceiling: int | None = None
text: Literal["listing", "excerpt", "detail", "none"]
teaser: bool = False
complete: Literal["stated", "end", "whole", "never"]
filters: Filters = Filters()
places: Literal["one", "many", "copies"]
fields: Fields = Fields()
robots: str
empty: Literal["trusted", "suspect"] = "trusted"
key: Key | None = None
title: Literal["final", "provisional"] = "final"
class Ask(_Frozen):
"""What the runner wants from one series of requests. Only the
dimensions the reader declared are set; for a `one` filter each series
carries one value. With `ids`, `countries` and `places` hold the host's
ids from `facets`, not codes or names. `params` are a search's saved
params (and a resolved key under `"key"`). An empty `Ask()` is an
unfiltered read.
`remote=True` asks for remote roles. A reach of countries and remote
means "in these countries, or remote", which no host can ask in one
request (two facets intersect), so the runner makes the remote roles a
series of their own: a reader never gets both `remote=True` and a
country or a place. With `remote="ids"`, `remote_ids` holds the host's
ids of its remote values (`facets(...)["remote"]`, any key), and
`remote` stays True."""
countries: tuple[str, ...] = ()
places: tuple[str, ...] = ()
remote: bool | None = None
remote_ids: tuple[str, ...] = ()
query: str = ""
params: dict[str, Any] = {}
class Place(TypedDict):
"""One place of a role, as the host states it: `text` the host's own
words ("Berlin, Germany"), `remote` True for a remote place ("Remote
(DE)"), False for an office, None when the host does not say. Resolving
`text` to a city, region or country is the app's job, not the reader's."""
text: str
remote: bool | None
class Row(TypedDict, total=False):
"""One role as a reader hands it over.
`source_id` is the host's id, unique within the board or the search;
`url` a web URL (`_shared.is_web_url`); `places` every place the host
states, never multiplied into rows, never cut to one; `teaser` the
short text the list carries. `location` is the host's own one-line
place, kept while storage reads it. `posted` is an aware datetime or
None; `description` plain text; `remote` True, False, a word or None;
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
`employment` only where the host states them, as `Fields` declares. Any
other stated key is kept in `raw` by the runner.
"""
source_id: Required[str]
title: Required[str]
url: Required[str]
places: Required[list[Place]]
teaser: str
location: str
posted: Any
department: str
description: str
remote: bool | str | None
salary_text: str
salary_min: int | None
salary_max: int | None
currency: str | None
country: str | None
seniority: str | None
employment: str | None
#: What a declared module's functions take, positionally.
SIGNATURES: dict[str, tuple[str, ...]] = {
"feed_url": ("slug", "ask"),
"parse": ("body", "slug"),
"page": ("get", "slug", "cursor", "ask"),
"rows": ("decoded", "slug"),
"detail": ("get", "slug", "row"),
"facets": ("get", "slug"),
}
def _takes(function: Callable[..., Any], count: int) -> bool:
"""Whether `function` accepts `count` positional arguments."""
try:
signature = inspect.signature(function)
except (TypeError, ValueError):
return True
try:
signature.bind(*([None] * count))
except TypeError:
return False
return True
def problems(module: ModuleType) -> list[str]:
"""What `module`'s `READS` gets wrong, on its own or against the
module's functions; empty for a module that declares none."""
if not hasattr(module, "READS"):
return []
reads = module.READS
if not isinstance(reads, Reads):
return ["READS is not a Reads"]
has = lambda name: callable(getattr(module, name, None)) # noqa: E731
out: list[str] = []
if reads.listing == "paged" and not (has("page") and has("rows")):
out.append('listing="paged" without page and rows')
if reads.listing == "whole" and not (has("feed_url") and has("parse")):
out.append('listing="whole" without feed_url and parse')
if reads.listing == "whole" and has("page"):
out.append('listing="whole" with page')
if reads.listing == "whole" and (reads.page_size is not None or reads.ceiling is not None):
out.append('page_size and ceiling are for listing="paged"')
if reads.page_size is not None and reads.page_size < 1:
out.append("page_size is not positive")
if reads.ceiling is not None and reads.ceiling < (reads.page_size or 1):
out.append("ceiling is below page_size")
if has("fallback_url") and reads.listing != "whole":
out.append('fallback_url without listing="whole"')
if reads.text == "detail" and not has("detail"):
out.append('text="detail" without detail')
if has("detail") and reads.text != "detail":
out.append(f'detail with text="{reads.text}"')
if reads.title == "provisional" and not has("detail"):
out.append('title="provisional" without detail to fill it')
if reads.teaser and reads.text != "detail":
out.append('teaser is for text="detail"; the list of "excerpt" is the teaser')
if (reads.complete == "whole") != (reads.listing == "whole"):
out.append('complete="whole" goes with listing="whole" and only with it')
if reads.scope == "search" and reads.complete not in ("never", "stated"):
out.append('scope="search" needs complete="never" or "stated"')
if reads.scope == "board" and reads.complete == "never":
out.append('scope="board" cannot be complete="never"')
ids = [name for name, reach in reads.filters.declared().items() if reach == "ids"]
if ids and not has("facets"):
out.append(f"an ids filter ({', '.join(ids)}) without facets")
if has("facets") and not ids:
out.append("facets without an ids filter")
if reads.filters.query == "ids":
out.append('query cannot be "ids"')
if key := reads.key:
if not _CONFIG_NAME.fullmatch(key.name):
out.append(f"key names no config entry ({key.name!r})")
if not key.fields:
out.append("key has no fields")
if key.env and len(key.env) != len(key.fields):
out.append("key.env does not match key.fields")
if any(not _ENV_NAME.fullmatch(name) for name in key.env):
out.append("key.env has a name that is not an environment variable")
if not reads.robots.strip():
out.append("robots is empty: record the robots.txt decision and its date")
for name, params in SIGNATURES.items():
if has(name) and not _takes(getattr(module, name), len(params)):
out.append(f"{name} does not take ({', '.join(params)})")
return out