Require READS, search params and a dated robots record; record robots.txt per requested path

Every reader declares, so undeclared(), the scope() fallback, robots_page and
ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may
carry company, places gains none. Each robots record names the paths the
reader requests, from robots.txt read again on 2026-09-30. Platsbanken states
remote, HR Manager states no place.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:51:13 +02:00
1 parent 4e827665dd
commit 2349fc424b
46 files changed
+323 -100

No files matched your search

+14 -22
View File
@@ -241,7 +241,7 @@ from . import _contract
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
"Place", "Reads", "Row", "load", "problems", "scope", "undeclared"]
"Place", "Reads", "Row", "load", "problems"]
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
@@ -250,27 +250,28 @@ DEFAULT_RANK = 100
#: What every board module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: What a module of `scope="search"` defines.
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed")
#: What a module of `scope="search"` defines: `PARAMS` and `clean_params` are
#: what `app/jobs/sources.search_params` stores a search's params with.
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed", "PARAMS", "clean_params")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok",
"fallback_url", "detail", "facets", "clean_params", "probe_url", "probe_ok",
"probe_slugs")
def scope(module: ModuleType) -> str:
"""`board` or `search`; `board` for a module without `READS`."""
def _search(module: ModuleType) -> bool:
reads = getattr(module, "READS", None)
return reads.scope if isinstance(reads, Reads) else "board"
return isinstance(reads, Reads) and reads.scope == "search"
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal. `READS` is checked when the
module declares one (`_contract.problems`)."""
required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED
in the package, which `NAME` must equal. A module without `READS` is
held to a board's names and fails on the missing declaration; the
declaration itself is `_contract.problems`."""
required = REQUIRED_SEARCH if _search(module) else REQUIRED
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
@@ -278,8 +279,6 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]:
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
out.append("ROBOTS_ANY_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
@@ -323,20 +322,13 @@ def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
return {module.NAME: module for module in modules}
def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]:
"""The names of the modules (of `READERS` by default) without `READS`,
read with the older signatures until they declare."""
return [name for name, module in (READERS if modules is None else modules).items()
if not hasattr(module, "READS")]
#: name -> module for every reader, in order of `(RANK, NAME)`.
READERS: dict[str, ModuleType] = load(__path__, __name__)
#: host -> its module: the readers of `scope="board"` and the undeclared.
#: host -> its module: the readers of `scope="board"`.
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if scope(module) == "board"}
if module.READS.scope == "board"}
#: provider -> its module: the readers of `scope="search"`.
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if scope(module) == "search"}
if module.READS.scope == "search"}