Move the eight hosts into a module each under plugins/atshosts and build the ats plugin's tables from them
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0a1d3c67c7
commit
b02b72b4aa
12 files changed
+1296
-789
No files matched your search
@@ -0,0 +1,113 @@
|
||||
"""The hosts the `ats` plugin reads, one module each.
|
||||
|
||||
A host registers by existing: every module in this package whose name does
|
||||
not start with `_` is loaded, in order of its name, and checked against the
|
||||
contract below when the package is imported, so a module that breaks it fails
|
||||
the import with a message naming the module and what it lacks, rather than a
|
||||
pull that fails board by board. `plugins/ats.py` builds its allow-list,
|
||||
feeds and pages from `HOSTS`, and `app/jobs/discover.py` its patterns and
|
||||
probes. The package sits beside `ats.py` rather than being `plugins/ats/`,
|
||||
since `desk.sources.registry` loads `plugins/*.py` by path and the tests
|
||||
patch names on the `ats` module.
|
||||
|
||||
A module is named after its host and holds:
|
||||
|
||||
NAME = "lever" # the host of a target `lever:<slug>`, and the module's name
|
||||
LABEL = "Lever" # what the Sources page's select shows
|
||||
PATTERNS = (r"...",) # discovery: over a URL; the slug is group 1, or the
|
||||
# non-empty groups joined by "/" when there are more
|
||||
|
||||
def slug_ok(slug) -> bool # strict; a slug that fails is no target
|
||||
def allowed(slug) -> Callable[[str], bool] # the host names a fetch may reach
|
||||
def careers_url(slug) -> str # the page a person opens
|
||||
|
||||
and one of two ways to read a board:
|
||||
|
||||
def feed_url(slug) -> str # the whole board in one answer,
|
||||
def parse(body: bytes, slug) -> list[Row] # read with `parse`
|
||||
|
||||
def page(get: Get, slug, cursor) -> Page # a paged board, one page a call,
|
||||
def rows(decoded, slug) -> list[Row] # its rows from `page`'s decoded page
|
||||
|
||||
and, optionally:
|
||||
|
||||
def fallback_url(slug) -> str # a smaller feed, read once when
|
||||
# `feed_url`'s answer is over the byte cap
|
||||
def robots_page(slug) -> str # the URL the site's robots.txt must allow
|
||||
def detail(get: Get, slug, row: Row) -> Row # the row filled from the role's own page
|
||||
def probe_url(slug) -> str # discovery: answers 2xx only for a board
|
||||
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
||||
def probe_slugs(guess) -> tuple[str, ...] # the slugs a guessed name is probed as
|
||||
|
||||
`Get`, `Row` and `Page` are in `_shared`, with the helpers several hosts use.
|
||||
A host module imports nothing from `plugins.ats`: the fetch it is handed is
|
||||
`ats._get` bound to the board, so every request a host makes passes the same
|
||||
allow-list, redirect rule, byte cap and deadline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from types import ModuleType
|
||||
|
||||
#: What every host module defines.
|
||||
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
||||
|
||||
#: The functions a module may define; each must be callable if present.
|
||||
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
||||
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
|
||||
|
||||
|
||||
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
||||
"""What `module` lacks or gets wrong against the contract, one phrase
|
||||
each; empty for a module that keeps it. `name` is the module's own name
|
||||
in the package, which `NAME` must equal."""
|
||||
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
|
||||
if name is not None and getattr(module, "NAME", name) != name:
|
||||
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
||||
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
|
||||
out.append("LABEL is not a non-empty string")
|
||||
patterns = getattr(module, "PATTERNS", ())
|
||||
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
|
||||
out.append("PATTERNS is not a tuple of strings")
|
||||
else:
|
||||
for pattern in patterns:
|
||||
try:
|
||||
re.compile(pattern)
|
||||
except (TypeError, re.error) as exc:
|
||||
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
|
||||
out += [f"{attr} is not callable" for attr in FUNCTIONS
|
||||
if hasattr(module, attr) and not callable(getattr(module, attr))]
|
||||
if hasattr(module, "page"):
|
||||
if not hasattr(module, "rows"):
|
||||
out.append("page without rows")
|
||||
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
|
||||
out.append("neither feed_url and parse nor page and rows")
|
||||
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
|
||||
out.append("probe_ok without probe_url")
|
||||
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
|
||||
out.append("probe_slugs without probe_url")
|
||||
return out
|
||||
|
||||
|
||||
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
|
||||
"""`NAME -> module` for every module of `package` (found on `path`) whose
|
||||
name does not start with `_`, sorted by name. Raises ImportError naming
|
||||
the first module that breaks the contract."""
|
||||
hosts: dict[str, ModuleType] = {}
|
||||
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
|
||||
if info.name.startswith("_"):
|
||||
continue
|
||||
module = importlib.import_module(f"{package}.{info.name}")
|
||||
if found := problems(module, info.name):
|
||||
raise ImportError(f"{package}.{info.name} breaks the host contract: "
|
||||
+ "; ".join(found))
|
||||
hosts[module.NAME] = module
|
||||
return hosts
|
||||
|
||||
|
||||
#: host -> its module, in order of name.
|
||||
HOSTS: dict[str, ModuleType] = load(__path__, __name__)
|
||||
Reference in new issue
Block a user