Require READS, search params and a dated robots record; record robots.txt per requested path
Every reader declares, so undeclared(), the scope() fallback, robots_page and ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may carry company, places gains none. Each robots record names the paths the reader requests, from robots.txt read again on 2026-09-30. Platsbanken states remote, HR Manager states no place. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
4e827665dd
commit
2349fc424b
46 files changed
+323
-100
No files matched your search
+14
-22
@@ -241,7 +241,7 @@ from . import _contract
|
||||
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
|
||||
|
||||
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
|
||||
"Place", "Reads", "Row", "load", "problems", "scope", "undeclared"]
|
||||
"Place", "Reads", "Row", "load", "problems"]
|
||||
|
||||
#: The rank of a module that states none: after the eight first hosts, whose
|
||||
#: ranks (10 to 80) keep the order they had before there were modules.
|
||||
@@ -250,27 +250,28 @@ DEFAULT_RANK = 100
|
||||
#: What every board module defines.
|
||||
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
||||
|
||||
#: What a module of `scope="search"` defines.
|
||||
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed")
|
||||
#: What a module of `scope="search"` defines: `PARAMS` and `clean_params` are
|
||||
#: what `app/jobs/sources.search_params` stores a search's params with.
|
||||
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed", "PARAMS", "clean_params")
|
||||
|
||||
#: The functions a module may define; each must be callable if present.
|
||||
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
||||
"fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok",
|
||||
"fallback_url", "detail", "facets", "clean_params", "probe_url", "probe_ok",
|
||||
"probe_slugs")
|
||||
|
||||
|
||||
def scope(module: ModuleType) -> str:
|
||||
"""`board` or `search`; `board` for a module without `READS`."""
|
||||
def _search(module: ModuleType) -> bool:
|
||||
reads = getattr(module, "READS", None)
|
||||
return reads.scope if isinstance(reads, Reads) else "board"
|
||||
return isinstance(reads, Reads) and reads.scope == "search"
|
||||
|
||||
|
||||
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
||||
"""What `module` lacks or gets wrong against the contract, one phrase
|
||||
each; empty for a module that keeps it. `name` is the module's own name
|
||||
in the package, which `NAME` must equal. `READS` is checked when the
|
||||
module declares one (`_contract.problems`)."""
|
||||
required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED
|
||||
in the package, which `NAME` must equal. A module without `READS` is
|
||||
held to a board's names and fails on the missing declaration; the
|
||||
declaration itself is `_contract.problems`."""
|
||||
required = REQUIRED_SEARCH if _search(module) else REQUIRED
|
||||
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
|
||||
if name is not None and getattr(module, "NAME", name) != name:
|
||||
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
||||
@@ -278,8 +279,6 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
||||
out.append("LABEL is not a non-empty string")
|
||||
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
||||
out.append("KEEP_CASE is not a bool")
|
||||
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
|
||||
out.append("ROBOTS_ANY_CASE is not a bool")
|
||||
rank = getattr(module, "RANK", DEFAULT_RANK)
|
||||
if not isinstance(rank, int) or isinstance(rank, bool):
|
||||
out.append("RANK is not an int")
|
||||
@@ -323,20 +322,13 @@ def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
|
||||
return {module.NAME: module for module in modules}
|
||||
|
||||
|
||||
def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]:
|
||||
"""The names of the modules (of `READERS` by default) without `READS`,
|
||||
read with the older signatures until they declare."""
|
||||
return [name for name, module in (READERS if modules is None else modules).items()
|
||||
if not hasattr(module, "READS")]
|
||||
|
||||
|
||||
#: name -> module for every reader, in order of `(RANK, NAME)`.
|
||||
READERS: dict[str, ModuleType] = load(__path__, __name__)
|
||||
|
||||
#: host -> its module: the readers of `scope="board"` and the undeclared.
|
||||
#: host -> its module: the readers of `scope="board"`.
|
||||
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
|
||||
if scope(module) == "board"}
|
||||
if module.READS.scope == "board"}
|
||||
|
||||
#: provider -> its module: the readers of `scope="search"`.
|
||||
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
|
||||
if scope(module) == "search"}
|
||||
if module.READS.scope == "search"}
|
||||
Reference in new issue
Block a user