Require READS, search params and a dated robots record; record robots.txt per requested path
Every reader declares, so undeclared(), the scope() fallback, robots_page and ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may carry company, places gains none. Each robots record names the paths the reader requests, from robots.txt read again on 2026-09-30. Platsbanken states remote, HR Manager states no place. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
4e827665dd
commit
2349fc424b
46 files changed
+323
-100
No files matched your search
@@ -20,14 +20,14 @@ Signatures of a module that declares `READS`, with `Get` and `Page` from
|
||||
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
|
||||
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
|
||||
def clean_params(params: dict[str, Any]) -> dict[str, Any] # scope="search"
|
||||
|
||||
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
|
||||
`country`, a place name for `place`, any word for `remote`. A dimension the
|
||||
host has no facet for is left out, and the runner then reads it unfiltered;
|
||||
a dimension present without a wanted value means the host has none of it.
|
||||
|
||||
A module without `READS` keeps the signatures without `ask` until it
|
||||
declares; `undeclared()` in the package lists those modules.
|
||||
A module without `READS` fails the import: every reader declares.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -57,6 +57,9 @@ _CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
|
||||
#: An environment variable name.
|
||||
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
|
||||
|
||||
#: The date a robots.txt record must carry: when the file was read.
|
||||
_DATE = re.compile(r"20\d\d-\d\d-\d\d")
|
||||
|
||||
|
||||
class _Frozen(Model):
|
||||
"""A declaration: immutable, extra keys refused."""
|
||||
@@ -123,11 +126,13 @@ class Reads(_Frozen):
|
||||
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
|
||||
`whole` (the one answer), `never` (a search). `places`: `one` (the host
|
||||
states one), `many` (a list per role), `copies` (the host lists one
|
||||
posting per place, each with its own id). `robots`: the robots.txt
|
||||
decision, with the date, taken when the reader was built. `empty`:
|
||||
`suspect` when an unknown slug answers an empty list rather than an
|
||||
error. `title`: `provisional` when the listing may not state the title
|
||||
(a sitemap), which detail then fills.
|
||||
posting per place, each with its own id), `none` (the host states no
|
||||
place; every row has `places == []`). `robots`: the robots.txt
|
||||
decision for the paths the reader requests, with the date the file was
|
||||
read. `empty`: `suspect` when an unknown slug answers an empty list
|
||||
rather than an error. `title`: `provisional` when the listing may not
|
||||
state the title (a sitemap), which detail then fills; nothing reads it
|
||||
yet, since `_runner.DETAIL_FIELDS` has no `title`.
|
||||
"""
|
||||
|
||||
scope: Literal["board", "search"]
|
||||
@@ -138,7 +143,7 @@ class Reads(_Frozen):
|
||||
teaser: bool = False
|
||||
complete: Literal["stated", "end", "whole", "never"]
|
||||
filters: Filters = Filters()
|
||||
places: Literal["one", "many", "copies"]
|
||||
places: Literal["one", "many", "copies", "none"]
|
||||
fields: Fields = Fields()
|
||||
robots: str
|
||||
empty: Literal["trusted", "suspect"] = "trusted"
|
||||
@@ -196,8 +201,10 @@ class Row(TypedDict, total=False):
|
||||
place, kept while storage reads it. `posted` is an aware datetime or
|
||||
None; `description` plain text; `remote` True, False, a word or None;
|
||||
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
|
||||
`employment` only where the host states them, as `Fields` declares. Any
|
||||
other stated key is kept in `raw` by the runner.
|
||||
`employment` only where the host states them, as `Fields` declares.
|
||||
`company` is the employer's name on a search row, which lists many
|
||||
(`plugins/search.py` stores it); a board row carries none, the board is
|
||||
the company. Any other stated key is kept in `raw` by the runner.
|
||||
"""
|
||||
|
||||
source_id: Required[str]
|
||||
@@ -217,6 +224,7 @@ class Row(TypedDict, total=False):
|
||||
country: str | None
|
||||
seniority: str | None
|
||||
employment: str | None
|
||||
company: str
|
||||
|
||||
|
||||
#: What a declared module's functions take, positionally.
|
||||
@@ -227,6 +235,7 @@ SIGNATURES: dict[str, tuple[str, ...]] = {
|
||||
"rows": ("decoded", "slug"),
|
||||
"detail": ("get", "slug", "row"),
|
||||
"facets": ("get", "slug"),
|
||||
"clean_params": ("params",),
|
||||
}
|
||||
|
||||
|
||||
@@ -245,9 +254,9 @@ def _takes(function: Callable[..., Any], count: int) -> bool:
|
||||
|
||||
def problems(module: ModuleType) -> list[str]:
|
||||
"""What `module`'s `READS` gets wrong, on its own or against the
|
||||
module's functions; empty for a module that declares none."""
|
||||
module's functions; "no READS" for a module that declares none."""
|
||||
if not hasattr(module, "READS"):
|
||||
return []
|
||||
return ["no READS: every reader declares what it reads"]
|
||||
reads = module.READS
|
||||
if not isinstance(reads, Reads):
|
||||
return ["READS is not a Reads"]
|
||||
@@ -305,6 +314,17 @@ def problems(module: ModuleType) -> list[str]:
|
||||
|
||||
if not reads.robots.strip():
|
||||
out.append("robots is empty: record the robots.txt decision and its date")
|
||||
elif not _DATE.search(reads.robots):
|
||||
out.append("robots has no date: record when robots.txt was read (YYYY-MM-DD)")
|
||||
|
||||
if reads.scope == "search" and hasattr(module, "PARAMS"):
|
||||
params = module.PARAMS
|
||||
if not isinstance(params, dict) or not params:
|
||||
out.append("PARAMS is not a dict of saved param -> filter dimension")
|
||||
else:
|
||||
dimensions = set(Filters.model_fields)
|
||||
out += [f"PARAMS maps {param!r} to {dimension!r}, not a filter dimension"
|
||||
for param, dimension in params.items() if dimension not in dimensions]
|
||||
|
||||
for name, params in SIGNATURES.items():
|
||||
if has(name) and not _takes(getattr(module, name), len(params)):
|
||||
|
||||
Reference in new issue
Block a user