Files
jobboard/plugins/atshosts/__init__.py
T

124 lines
6.1 KiB
Python

"""The hosts the `ats` plugin reads, one module each.
A host registers by existing: every module in this package whose name does
not start with `_` is loaded, in order of its name, and checked against the
contract below when the package is imported, so a module that breaks it fails
the import with a message naming the module and what it lacks, rather than a
pull that fails board by board. `plugins/ats.py` builds its allow-list,
feeds and pages from `HOSTS`, and `app/jobs/discover.py` its patterns and
probes. The package sits beside `ats.py` rather than being `plugins/ats/`,
since `desk.sources.registry` loads `plugins/*.py` by path and the tests
patch names on the `ats` module.
A module is named after its host and holds:
NAME = "lever" # the host of a target `lever:<slug>`, and the module's name
LABEL = "Lever" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: over a URL; the slug is group 1, or the
# non-empty groups joined by "/" when there are more
def slug_ok(slug) -> bool # strict; a slug that fails is no target
def allowed(slug) -> Callable[[str], bool] # the host names a fetch may reach
def careers_url(slug) -> str # the page a person opens
and one of two ways to read a board:
def feed_url(slug) -> str # the whole board in one answer,
def parse(body: bytes, slug) -> list[Row] # read with `parse`
def page(get: Get, slug, cursor) -> Page # a paged board, one page a call,
def rows(decoded, slug) -> list[Row] # its rows from `page`'s decoded page
and, optionally:
def fallback_url(slug) -> str # a smaller feed, read once when
# `feed_url`'s answer is over the byte cap
def robots_page(slug) -> str # the URL the site's robots.txt must allow
def detail(get: Get, slug, row: Row) -> Row # the row filled from the role's own page
def probe_url(slug) -> str # discovery: answers 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess) -> tuple[str, ...] # the slugs a guessed name is probed as
`Get`, `Row` and `Page` are in `_shared`, with the helpers several hosts use.
A host module imports nothing from `plugins.ats`: the fetch it is handed is
`ats._get` bound to the board, so every request a host makes passes the same
allow-list, redirect rule, byte cap and deadline.
"""
from __future__ import annotations
import importlib
import pkgutil
import re
from collections.abc import Iterable
from types import ModuleType
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
DEFAULT_RANK = 100
#: What every host module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal."""
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
patterns = getattr(module, "PATTERNS", ())
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
out.append("PATTERNS is not a tuple of strings")
else:
for pattern in patterns:
try:
re.compile(pattern)
except (TypeError, re.error) as exc:
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
out += [f"{attr} is not callable" for attr in FUNCTIONS
if hasattr(module, attr) and not callable(getattr(module, attr))]
if hasattr(module, "page"):
if not hasattr(module, "rows"):
out.append("page without rows")
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
out.append("neither feed_url and parse nor page and rows")
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
out.append("probe_ok without probe_url")
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
out.append("probe_slugs without probe_url")
return out
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
"""`NAME -> module` for every module of `package` (found on `path`) whose
name does not start with `_`, in order of `(RANK, NAME)`. Raises
ImportError naming the first module that breaks the contract."""
modules: list[ModuleType] = []
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{package}.{info.name}")
if found := problems(module, info.name):
raise ImportError(f"{package}.{info.name} breaks the host contract: "
+ "; ".join(found))
modules.append(module)
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
return {module.NAME: module for module in modules}
#: host -> its module, in order of `(RANK, NAME)`.
HOSTS: dict[str, ModuleType] = load(__path__, __name__)