Rename plugins/atshosts to plugins/readers
Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
af337ef558
commit
73dec9028e
48 files changed
+104
-104
No files matched your search
@@ -0,0 +1,217 @@
|
||||
"""The hosts the `ats` plugin reads, one module each.
|
||||
|
||||
A host registers by existing: every module in this package whose name does
|
||||
not start with `_` is loaded when the package is imported, checked against
|
||||
the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A
|
||||
module that breaks the contract fails the import with an ImportError naming
|
||||
the module and what it lacks, rather than a pull that fails board by board.
|
||||
`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host
|
||||
select and its description from `HOSTS`, and `app/jobs/discover.py` its
|
||||
patterns and probes. The package sits beside `ats.py` rather than being
|
||||
`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and
|
||||
the tests patch names on the `ats` module.
|
||||
|
||||
A module is named after its host (`plugins/readers/workday.py`) and imports
|
||||
`desk` and `._shared` only, never `plugins.ats`. It holds:
|
||||
|
||||
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
||||
LABEL = "Workday" # what the Sources page's select shows
|
||||
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||
RANK = 100 # optional int; lower comes first in the select and the probes
|
||||
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
||||
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
|
||||
|
||||
def slug_ok(slug: str) -> bool # strict, see below
|
||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
||||
def careers_url(slug: str) -> str # the page a person opens
|
||||
|
||||
and one of two ways to read a board:
|
||||
|
||||
def feed_url(slug: str) -> str # the whole board in one answer,
|
||||
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call,
|
||||
def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page
|
||||
|
||||
and, optionally:
|
||||
|
||||
def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read
|
||||
# once when `feed_url` is over the cap
|
||||
def robots_page(slug: str) -> str # the URL robots.txt must allow
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page
|
||||
def probe_url(slug: str) -> str # discovery: 2xx only for a board
|
||||
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
||||
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
|
||||
|
||||
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
|
||||
hold 10 to 80, which keeps the order they had before there were modules, so
|
||||
a new host comes after them unless it says otherwise. The order decides the
|
||||
select's default and which host wins when discovery's guessed slug exists on
|
||||
two.
|
||||
|
||||
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
|
||||
which lets a trailing newline through) and refuses anything that could add a
|
||||
path, a query, a fragment, a port or a user to the URLs built from it.
|
||||
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
|
||||
(one DNS label) cover the usual cases; a slug with several parts joins them
|
||||
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
|
||||
|
||||
**Patterns.** The slug is group 1, or, when a pattern has more than one
|
||||
group, the groups that matched joined by "/". Only group 1 is checked
|
||||
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
|
||||
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
|
||||
paths are case-sensitive (Workday's site), since robots.txt paths are too. A
|
||||
pattern without a group names the host with no slug. A host that
|
||||
`discover.py` names in its own `_UNREADABLE` table is taken from its module
|
||||
instead as soon as the module exists.
|
||||
|
||||
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
||||
name; `_shared.only(host)` is the usual one. Every request goes through
|
||||
`get`, which is `ats._get` bound to the board:
|
||||
|
||||
get(url, *, method="GET", payload=None, headers=None) -> bytes
|
||||
|
||||
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
|
||||
default User-Agent and `Accept: application/json, text/xml`, and replace
|
||||
either where they name it. `get` is https only on port 443, never to an IP
|
||||
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
|
||||
most three redirects each checked against `allowed(slug)`, cuts the body at
|
||||
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
|
||||
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
|
||||
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
|
||||
failure, and another exception (`ValueError` among them) for a refused host,
|
||||
an oversized body or a broken redirect. A module lets these through; the
|
||||
shared code logs, records and decides.
|
||||
|
||||
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
|
||||
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
|
||||
id, to decide completeness), the total the host states or None, and the
|
||||
cursor of the next page or None after the last. The first call gets cursor
|
||||
None. A board with a total is complete when as many distinct ids were read;
|
||||
one without, when the read ended on an empty page. A 429 after the first page
|
||||
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
|
||||
apart. A paged module may also have `parse(body, slug)` for one page's body.
|
||||
|
||||
**Rows.** A `Row` is a dict with `source_id` (the host's own id for the
|
||||
role, unique on the board), `title`, `location`, `url` (http or https,
|
||||
checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see
|
||||
`_shared.parse_dt`), `department`, `description` (plain text, see
|
||||
`desk.sources.strip_html`), `remote` (True, False, a word or None) and
|
||||
`salary_text`; and, where the host states pay, `salary_min`, `salary_max`
|
||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`.
|
||||
|
||||
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
||||
dict of the fields it found. The shared code calls it for roles not yet read
|
||||
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
|
||||
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
|
||||
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
|
||||
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
|
||||
and `posted`, the keys whose columns survive a later pull that sends the
|
||||
listing's empty value; any other key is left out with a debug log line.
|
||||
`title`, `url`, `department` and `salary_text` cannot be filled from
|
||||
`detail`: the listing rewrites them every pull. A role not read in detail
|
||||
on a pull is sent with an empty description, so the one read earlier stays.
|
||||
A 404 or a 410 counts as read; any other failure is logged and retried on a
|
||||
later pull; a 429 ends the detail reads of that board for the pull.
|
||||
A role read in detail on an earlier pull is sent without the
|
||||
listing's place and remote, so what its own page said about them stays.
|
||||
Completeness is decided by the listing alone.
|
||||
|
||||
**robots.txt.** With `robots_page(slug)`, the shared code reads
|
||||
`https://<its host>/robots.txt` through `get` before the listing, at most
|
||||
once a day per board (`ats:robots:<target>`), following a redirect to the
|
||||
host's `www.` twin or back and no other. The board fails with "robots.txt
|
||||
disallows it" when the rules for `*` or for the app's own token disallow the
|
||||
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
|
||||
final `$` in paths, case-sensitive paths, or without case for a host with
|
||||
`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as
|
||||
Workday does). A 4xx other than 429 means no
|
||||
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
|
||||
the board with "robots.txt could not be read" and is asked again next pull.
|
||||
`robots_page` must be on a host `allowed(slug)` admits.
|
||||
|
||||
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
||||
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
||||
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
|
||||
name is probed as, `(guess,)` without it. A host without `probe_url` is never
|
||||
probed, only found by a link.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from types import ModuleType
|
||||
|
||||
#: The rank of a module that states none: after the eight first hosts, whose
|
||||
#: ranks (10 to 80) keep the order they had before there were modules.
|
||||
DEFAULT_RANK = 100
|
||||
|
||||
#: What every host module defines.
|
||||
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
||||
|
||||
#: The functions a module may define; each must be callable if present.
|
||||
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
||||
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
|
||||
|
||||
|
||||
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
||||
"""What `module` lacks or gets wrong against the contract, one phrase
|
||||
each; empty for a module that keeps it. `name` is the module's own name
|
||||
in the package, which `NAME` must equal."""
|
||||
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
|
||||
if name is not None and getattr(module, "NAME", name) != name:
|
||||
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
||||
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
|
||||
out.append("LABEL is not a non-empty string")
|
||||
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
||||
out.append("KEEP_CASE is not a bool")
|
||||
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
|
||||
out.append("ROBOTS_ANY_CASE is not a bool")
|
||||
rank = getattr(module, "RANK", DEFAULT_RANK)
|
||||
if not isinstance(rank, int) or isinstance(rank, bool):
|
||||
out.append("RANK is not an int")
|
||||
patterns = getattr(module, "PATTERNS", ())
|
||||
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
|
||||
out.append("PATTERNS is not a tuple of strings")
|
||||
else:
|
||||
for pattern in patterns:
|
||||
try:
|
||||
re.compile(pattern)
|
||||
except (TypeError, re.error) as exc:
|
||||
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
|
||||
out += [f"{attr} is not callable" for attr in FUNCTIONS
|
||||
if hasattr(module, attr) and not callable(getattr(module, attr))]
|
||||
if hasattr(module, "page"):
|
||||
if not hasattr(module, "rows"):
|
||||
out.append("page without rows")
|
||||
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
|
||||
out.append("neither feed_url and parse nor page and rows")
|
||||
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
|
||||
out.append("probe_ok without probe_url")
|
||||
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
|
||||
out.append("probe_slugs without probe_url")
|
||||
return out
|
||||
|
||||
|
||||
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
|
||||
"""`NAME -> module` for every module of `package` (found on `path`) whose
|
||||
name does not start with `_`, in order of `(RANK, NAME)`. Raises
|
||||
ImportError naming the first module that breaks the contract."""
|
||||
modules: list[ModuleType] = []
|
||||
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
|
||||
if info.name.startswith("_"):
|
||||
continue
|
||||
module = importlib.import_module(f"{package}.{info.name}")
|
||||
if found := problems(module, info.name):
|
||||
raise ImportError(f"{package}.{info.name} breaks the host contract: "
|
||||
+ "; ".join(found))
|
||||
modules.append(module)
|
||||
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
|
||||
return {module.NAME: module for module in modules}
|
||||
|
||||
|
||||
#: host -> its module, in order of `(RANK, NAME)`.
|
||||
HOSTS: dict[str, ModuleType] = load(__path__, __name__)
|
||||
Reference in new issue
Block a user