"""The hosts the `ats` plugin reads, one module each. A host registers by existing: every module in this package whose name does not start with `_` is loaded, in order of its name, and checked against the contract below when the package is imported, so a module that breaks it fails the import with a message naming the module and what it lacks, rather than a pull that fails board by board. `plugins/ats.py` builds its allow-list, feeds and pages from `HOSTS`, and `app/jobs/discover.py` its patterns and probes. The package sits beside `ats.py` rather than being `plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and the tests patch names on the `ats` module. A module is named after its host and holds: NAME = "lever" # the host of a target `lever:`, and the module's name LABEL = "Lever" # what the Sources page's select shows PATTERNS = (r"...",) # discovery: over a URL; the slug is group 1, or the # non-empty groups joined by "/" when there are more def slug_ok(slug) -> bool # strict; a slug that fails is no target def allowed(slug) -> Callable[[str], bool] # the host names a fetch may reach def careers_url(slug) -> str # the page a person opens and one of two ways to read a board: def feed_url(slug) -> str # the whole board in one answer, def parse(body: bytes, slug) -> list[Row] # read with `parse` def page(get: Get, slug, cursor) -> Page # a paged board, one page a call, def rows(decoded, slug) -> list[Row] # its rows from `page`'s decoded page and, optionally: def fallback_url(slug) -> str # a smaller feed, read once when # `feed_url`'s answer is over the byte cap def robots_page(slug) -> str # the URL the site's robots.txt must allow def detail(get: Get, slug, row: Row) -> Row # the row filled from the role's own page def probe_url(slug) -> str # discovery: answers 2xx only for a board def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough def probe_slugs(guess) -> tuple[str, ...] # the slugs a guessed name is probed as `Get`, `Row` and `Page` are in `_shared`, with the helpers several hosts use. A host module imports nothing from `plugins.ats`: the fetch it is handed is `ats._get` bound to the board, so every request a host makes passes the same allow-list, redirect rule, byte cap and deadline. """ from __future__ import annotations import importlib import pkgutil import re from collections.abc import Iterable from types import ModuleType #: What every host module defines. REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url") #: The functions a module may define; each must be callable if present. FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows", "fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs") def problems(module: ModuleType, name: str | None = None) -> list[str]: """What `module` lacks or gets wrong against the contract, one phrase each; empty for a module that keeps it. `name` is the module's own name in the package, which `NAME` must equal.""" out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)] if name is not None and getattr(module, "NAME", name) != name: out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}") if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL): out.append("LABEL is not a non-empty string") patterns = getattr(module, "PATTERNS", ()) if isinstance(patterns, str) or not isinstance(patterns, Iterable): out.append("PATTERNS is not a tuple of strings") else: for pattern in patterns: try: re.compile(pattern) except (TypeError, re.error) as exc: out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})") out += [f"{attr} is not callable" for attr in FUNCTIONS if hasattr(module, attr) and not callable(getattr(module, attr))] if hasattr(module, "page"): if not hasattr(module, "rows"): out.append("page without rows") elif not (hasattr(module, "feed_url") and hasattr(module, "parse")): out.append("neither feed_url and parse nor page and rows") if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"): out.append("probe_ok without probe_url") if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"): out.append("probe_slugs without probe_url") return out def load(path: Iterable[str], package: str) -> dict[str, ModuleType]: """`NAME -> module` for every module of `package` (found on `path`) whose name does not start with `_`, sorted by name. Raises ImportError naming the first module that breaks the contract.""" hosts: dict[str, ModuleType] = {} for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name): if info.name.startswith("_"): continue module = importlib.import_module(f"{package}.{info.name}") if found := problems(module, info.name): raise ImportError(f"{package}.{info.name} breaks the host contract: " + "; ".join(found)) hosts[module.NAME] = module return hosts #: host -> its module, in order of name. HOSTS: dict[str, ModuleType] = load(__path__, __name__)