116 lines
5.7 KiB
Python
116 lines
5.7 KiB
Python
"""The hosts the `ats` plugin reads, one module each.
|
|
|
|
A host registers by existing: every module in this package whose name does
|
|
not start with `_` is loaded, in order of its name, and checked against the
|
|
contract below when the package is imported, so a module that breaks it fails
|
|
the import with a message naming the module and what it lacks, rather than a
|
|
pull that fails board by board. `plugins/ats.py` builds its allow-list,
|
|
feeds and pages from `HOSTS`, and `app/jobs/discover.py` its patterns and
|
|
probes. The package sits beside `ats.py` rather than being `plugins/ats/`,
|
|
since `desk.sources.registry` loads `plugins/*.py` by path and the tests
|
|
patch names on the `ats` module.
|
|
|
|
A module is named after its host and holds:
|
|
|
|
NAME = "lever" # the host of a target `lever:<slug>`, and the module's name
|
|
LABEL = "Lever" # what the Sources page's select shows
|
|
PATTERNS = (r"...",) # discovery: over a URL; the slug is group 1, or the
|
|
# non-empty groups joined by "/" when there are more
|
|
|
|
def slug_ok(slug) -> bool # strict; a slug that fails is no target
|
|
def allowed(slug) -> Callable[[str], bool] # the host names a fetch may reach
|
|
def careers_url(slug) -> str # the page a person opens
|
|
|
|
and one of two ways to read a board:
|
|
|
|
def feed_url(slug) -> str # the whole board in one answer,
|
|
def parse(body: bytes, slug) -> list[Row] # read with `parse`
|
|
|
|
def page(get: Get, slug, cursor) -> Page # a paged board, one page a call,
|
|
def rows(decoded, slug) -> list[Row] # its rows from `page`'s decoded page
|
|
|
|
and, optionally:
|
|
|
|
def fallback_url(slug) -> str # a smaller feed, read once when
|
|
# `feed_url`'s answer is over the byte cap
|
|
def robots_page(slug) -> str # the URL the site's robots.txt must allow
|
|
def detail(get: Get, slug, row: Row) -> Row # the row filled from the role's own page
|
|
def probe_url(slug) -> str # discovery: answers 2xx only for a board
|
|
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
|
def probe_slugs(guess) -> tuple[str, ...] # the slugs a guessed name is probed as
|
|
|
|
`Get`, `Row` and `Page` are in `_shared`, with the helpers several hosts use.
|
|
A host module imports nothing from `plugins.ats`: the fetch it is handed is
|
|
`ats._get` bound to the board, so every request a host makes passes the same
|
|
allow-list, redirect rule, byte cap and deadline.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import pkgutil
|
|
import re
|
|
from collections.abc import Iterable
|
|
from types import ModuleType
|
|
|
|
#: What every host module defines.
|
|
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
|
|
|
#: The functions a module may define; each must be callable if present.
|
|
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
|
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
|
|
|
|
|
|
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
|
"""What `module` lacks or gets wrong against the contract, one phrase
|
|
each; empty for a module that keeps it. `name` is the module's own name
|
|
in the package, which `NAME` must equal."""
|
|
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
|
|
if name is not None and getattr(module, "NAME", name) != name:
|
|
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
|
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
|
|
out.append("LABEL is not a non-empty string")
|
|
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
|
out.append("KEEP_CASE is not a bool")
|
|
patterns = getattr(module, "PATTERNS", ())
|
|
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
|
|
out.append("PATTERNS is not a tuple of strings")
|
|
else:
|
|
for pattern in patterns:
|
|
try:
|
|
re.compile(pattern)
|
|
except (TypeError, re.error) as exc:
|
|
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
|
|
out += [f"{attr} is not callable" for attr in FUNCTIONS
|
|
if hasattr(module, attr) and not callable(getattr(module, attr))]
|
|
if hasattr(module, "page"):
|
|
if not hasattr(module, "rows"):
|
|
out.append("page without rows")
|
|
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
|
|
out.append("neither feed_url and parse nor page and rows")
|
|
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
|
|
out.append("probe_ok without probe_url")
|
|
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
|
|
out.append("probe_slugs without probe_url")
|
|
return out
|
|
|
|
|
|
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
|
|
"""`NAME -> module` for every module of `package` (found on `path`) whose
|
|
name does not start with `_`, sorted by name. Raises ImportError naming
|
|
the first module that breaks the contract."""
|
|
hosts: dict[str, ModuleType] = {}
|
|
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
|
|
if info.name.startswith("_"):
|
|
continue
|
|
module = importlib.import_module(f"{package}.{info.name}")
|
|
if found := problems(module, info.name):
|
|
raise ImportError(f"{package}.{info.name} breaks the host contract: "
|
|
+ "; ".join(found))
|
|
hosts[module.NAME] = module
|
|
return hosts
|
|
|
|
|
|
#: host -> its module, in order of name.
|
|
HOSTS: dict[str, ModuleType] = load(__path__, __name__)
|