Require READS, search params and a dated robots record; record robots.txt per requested path

Every reader declares, so undeclared(), the scope() fallback, robots_page and
ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may
carry company, places gains none. Each robots record names the paths the
reader requests, from robots.txt read again on 2026-09-30. Platsbanken states
remote, HR Manager states no place.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:51:13 +02:00
1 parent 4e827665dd
commit 2349fc424b
46 files changed
+323 -100

No files matched your search

+32 -12
View File
@@ -20,14 +20,14 @@ Signatures of a module that declares `READS`, with `Get` and `Page` from
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
def clean_params(params: dict[str, Any]) -> dict[str, Any] # scope="search"
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
`country`, a place name for `place`, any word for `remote`. A dimension the
host has no facet for is left out, and the runner then reads it unfiltered;
a dimension present without a wanted value means the host has none of it.
A module without `READS` keeps the signatures without `ask` until it
declares; `undeclared()` in the package lists those modules.
A module without `READS` fails the import: every reader declares.
"""
from __future__ import annotations
@@ -57,6 +57,9 @@ _CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
#: An environment variable name.
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
#: The date a robots.txt record must carry: when the file was read.
_DATE = re.compile(r"20\d\d-\d\d-\d\d")
class _Frozen(Model):
"""A declaration: immutable, extra keys refused."""
@@ -123,11 +126,13 @@ class Reads(_Frozen):
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
`whole` (the one answer), `never` (a search). `places`: `one` (the host
states one), `many` (a list per role), `copies` (the host lists one
posting per place, each with its own id). `robots`: the robots.txt
decision, with the date, taken when the reader was built. `empty`:
`suspect` when an unknown slug answers an empty list rather than an
error. `title`: `provisional` when the listing may not state the title
(a sitemap), which detail then fills.
posting per place, each with its own id), `none` (the host states no
place; every row has `places == []`). `robots`: the robots.txt
decision for the paths the reader requests, with the date the file was
read. `empty`: `suspect` when an unknown slug answers an empty list
rather than an error. `title`: `provisional` when the listing may not
state the title (a sitemap), which detail then fills; nothing reads it
yet, since `_runner.DETAIL_FIELDS` has no `title`.
"""
scope: Literal["board", "search"]
@@ -138,7 +143,7 @@ class Reads(_Frozen):
teaser: bool = False
complete: Literal["stated", "end", "whole", "never"]
filters: Filters = Filters()
places: Literal["one", "many", "copies"]
places: Literal["one", "many", "copies", "none"]
fields: Fields = Fields()
robots: str
empty: Literal["trusted", "suspect"] = "trusted"
@@ -196,8 +201,10 @@ class Row(TypedDict, total=False):
place, kept while storage reads it. `posted` is an aware datetime or
None; `description` plain text; `remote` True, False, a word or None;
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
`employment` only where the host states them, as `Fields` declares. Any
other stated key is kept in `raw` by the runner.
`employment` only where the host states them, as `Fields` declares.
`company` is the employer's name on a search row, which lists many
(`plugins/search.py` stores it); a board row carries none, the board is
the company. Any other stated key is kept in `raw` by the runner.
"""
source_id: Required[str]
@@ -217,6 +224,7 @@ class Row(TypedDict, total=False):
country: str | None
seniority: str | None
employment: str | None
company: str
#: What a declared module's functions take, positionally.
@@ -227,6 +235,7 @@ SIGNATURES: dict[str, tuple[str, ...]] = {
"rows": ("decoded", "slug"),
"detail": ("get", "slug", "row"),
"facets": ("get", "slug"),
"clean_params": ("params",),
}
@@ -245,9 +254,9 @@ def _takes(function: Callable[..., Any], count: int) -> bool:
def problems(module: ModuleType) -> list[str]:
"""What `module`'s `READS` gets wrong, on its own or against the
module's functions; empty for a module that declares none."""
module's functions; "no READS" for a module that declares none."""
if not hasattr(module, "READS"):
return []
return ["no READS: every reader declares what it reads"]
reads = module.READS
if not isinstance(reads, Reads):
return ["READS is not a Reads"]
@@ -305,6 +314,17 @@ def problems(module: ModuleType) -> list[str]:
if not reads.robots.strip():
out.append("robots is empty: record the robots.txt decision and its date")
elif not _DATE.search(reads.robots):
out.append("robots has no date: record when robots.txt was read (YYYY-MM-DD)")
if reads.scope == "search" and hasattr(module, "PARAMS"):
params = module.PARAMS
if not isinstance(params, dict) or not params:
out.append("PARAMS is not a dict of saved param -> filter dimension")
else:
dimensions = set(Filters.model_fields)
out += [f"PARAMS maps {param!r} to {dimension!r}, not a filter dimension"
for param, dimension in params.items() if dimension not in dimensions]
for name, params in SIGNATURES.items():
if has(name) and not _takes(getattr(module, name), len(params)):