plugins/readers/_contract.py holds Reads, Filters, Fields, Key, Ask, Place and Row, and problems(), which checks a module's READS against its functions and signatures. The package loader runs it for modules that declare READS and lists the others with undeclared(); READERS holds every module, HOSTS the boards and the undeclared, PROVIDERS the searches. The package docstring documents the traits, Ask, facets, places, teaser and the robots.txt rule. tests/test_readers_contract.py runs over every reader: READS declared (xfail per module in UNDECLARED), declaration against functions, places="many" and teaser fixtures, plus unit tests of the check with fake modules. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
330 lines
18 KiB
Python
330 lines
18 KiB
Python
"""The readers: one module per source of roles, a host's boards or a search.
|
|
|
|
A reader registers by existing: every module in this package whose name does
|
|
not start with `_` is loaded when the package is imported, checked against
|
|
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
|
|
`HOSTS` holds those of `scope="board"` (and every module without `READS`),
|
|
`PROVIDERS` those of `scope="search"`. A module that breaks the contract
|
|
fails the import with an ImportError naming the module and every problem,
|
|
rather than a pull that fails board by board. `plugins/ats.py` builds its
|
|
allow-list, feeds, pages, the Sources page's host select and its
|
|
description from `HOSTS`, and `app/jobs/discover.py` its patterns and
|
|
probes. The package sits beside `ats.py` rather than being `plugins/ats/`,
|
|
since `desk.sources.registry` loads `plugins/*.py` by path and the tests
|
|
patch names on the `ats` module. The types are in `._contract`.
|
|
|
|
A module is named after its source (`plugins/readers/workday.py`) and
|
|
imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It
|
|
holds:
|
|
|
|
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
|
LABEL = "Workday" # what the Sources page's select shows
|
|
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
|
RANK = 100 # optional int; lower comes first in the select and the probes
|
|
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
|
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
|
|
|
|
READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000,
|
|
text="detail", complete="stated",
|
|
filters=Filters(country="ids"), places="many",
|
|
fields=Fields(posted="listed"),
|
|
robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)")
|
|
|
|
def slug_ok(slug: str) -> bool # strict, see below
|
|
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
|
def careers_url(slug: str) -> str # the page a person opens
|
|
|
|
and the functions its `READS` names:
|
|
|
|
def feed_url(slug: str, ask: Ask) -> str # listing="whole": one answer,
|
|
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
|
|
|
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged",
|
|
def rows(decoded: Any, slug: str) -> list[Row] # a page a call
|
|
|
|
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
|
|
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
|
|
|
|
and, optionally:
|
|
|
|
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
|
|
# read once when `feed_url` is over the cap
|
|
def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt)
|
|
def probe_url(slug: str) -> str # discovery: 2xx only for a board
|
|
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
|
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
|
|
|
|
A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its
|
|
slug is the provider's own (a search reads no board), and it is not
|
|
discovered. A module without `READS` is read as a board with the older
|
|
signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by
|
|
`undeclared()`; the conformance test marks it until it declares.
|
|
|
|
**Declaration.** `READS` states what the reader does; the runner does it.
|
|
`problems()` checks it against the functions: `paged` needs `page` and
|
|
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
|
|
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
|
|
`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key`
|
|
names a config entry, `robots` is not empty, and each function takes the
|
|
arguments above. The traits:
|
|
|
|
- `scope`: `board`, one company's whole list; `search`, many companies,
|
|
never a whole list, so nothing a search stops listing is closed by it.
|
|
- `listing`: `whole`, one request; `paged`, with `page_size` and `ceiling`,
|
|
the offset past which the host answers nothing whatever is asked. A board
|
|
above its ceiling is incomplete unless a filter brings it under.
|
|
- `text`: `listing`, the list has the full text; `excerpt`, a teaser and no
|
|
more; `detail`, one request a role; `none`. `teaser=True` with `detail`
|
|
says the list carries a teaser as well.
|
|
- `complete`: `stated`, the host gives a total; `end`, an empty page ends
|
|
the list; `whole`, the one answer is all; `never`.
|
|
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
|
|
`one` (one value a request), `many` (a list in one request), `ids` (host
|
|
ids looked up with `facets`).
|
|
- `places`: `one`, `many`, or `copies` when the host publishes one posting
|
|
per place with its own id.
|
|
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
|
|
posting date; `listed`, a date the host moves; `none`), and `remote`,
|
|
`country`, `seniority`, `employment` (`stated` or `none`).
|
|
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
|
|
and its environment fallbacks. The runner resolves it and passes it in
|
|
`Ask.params["key"]`.
|
|
- `robots`: the robots.txt decision, see below. Required.
|
|
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
|
|
far under the last count counts as failed.
|
|
- `title`: `provisional` when the listing may lack the title and `detail`
|
|
fills it.
|
|
|
|
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
|
|
wants from one series of requests. It sets only dimensions the reader
|
|
declared. With `one` it runs a series per value and unions the rows by
|
|
`source_id`. With `ids`, `countries` and `places` hold host ids, which the
|
|
runner maps from `facets(get, slug)`: `{"country": {"DE": "<host id>"},
|
|
"place": {...}}`, called at most once a day per board and kept in state.
|
|
`params` holds a search's saved params. `Ask()` is an unfiltered read. A
|
|
reader turns `Ask` into its host's query in `feed_url` or `page` and never
|
|
filters rows itself.
|
|
|
|
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
|
|
hold 10 to 80, which keeps the order they had before there were modules, so
|
|
a new host comes after them unless it says otherwise. The order decides the
|
|
select's default and which host wins when discovery's guessed slug exists on
|
|
two.
|
|
|
|
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
|
|
which lets a trailing newline through) and refuses anything that could add a
|
|
path, a query, a fragment, a port or a user to the URLs built from it.
|
|
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
|
|
(one DNS label) cover the usual cases; a slug with several parts joins them
|
|
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
|
|
|
|
**Patterns.** The slug is group 1, or, when a pattern has more than one
|
|
group, the groups that matched joined by "/". Only group 1 is checked
|
|
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
|
|
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
|
|
paths are case-sensitive (Workday's site). A pattern without a group names
|
|
the host with no slug. A host that `discover.py` names in its own
|
|
`_UNREADABLE` table is taken from its module instead as soon as the module
|
|
exists.
|
|
|
|
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
|
name; `_shared.only(host)` is the usual one. Every request goes through
|
|
`get`, which is `ats._get` bound to the board:
|
|
|
|
get(url, *, method="GET", payload=None, headers=None) -> bytes
|
|
|
|
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
|
|
default User-Agent and `Accept: application/json, text/xml`, and replace
|
|
either where they name it. `get` is https only on port 443, never to an IP
|
|
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
|
|
most three redirects each checked against `allowed(slug)`, cuts the body at
|
|
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
|
|
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
|
|
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
|
|
failure, and another exception (`ValueError` among them) for a refused host,
|
|
an oversized body or a broken redirect. A module lets these through; the
|
|
shared code logs, records and decides.
|
|
|
|
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
|
|
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
|
|
id, to decide completeness), the total the host states or None, and the
|
|
cursor of the next page or None after the last. The first call gets cursor
|
|
None. A board with a total is complete when as many distinct ids were read;
|
|
one without, when the read ended on an empty page. A 429 after the first page
|
|
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
|
|
apart. A paged module may also have `parse(body, slug)` for one page's body.
|
|
|
|
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
|
|
own id for the role, unique on the board), `title`, `url` (http or https,
|
|
checked with `_shared.is_web_url`) and `places`; `location` (the host's
|
|
one-line place, kept while storage reads it), `posted` (an aware datetime
|
|
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
|
|
see `desk.sources.strip_html`), `remote` (True, False, a word or None) and
|
|
`salary_text`; where the host states pay, `salary_min`, `salary_max`
|
|
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
|
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
|
`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a
|
|
new field needs no change there. Undeclared modules do not state `places`
|
|
yet.
|
|
|
|
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
|
remote=...)]`, one entry per place the host states, in the host's order, in
|
|
the host's words. A reader never multiplies a row per place, never drops a
|
|
place and never joins places into one `text`. `remote` is True for a remote
|
|
place, False for an office, None when unsaid. A host that publishes one
|
|
posting per place, each with its own id, declares `places="copies"`: each
|
|
copy is a row with its one place, and `dupes.py` folds them into one role
|
|
that keeps each copy's link.
|
|
|
|
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
|
|
over as `teaser`, plain text, whether or not the full text comes later. It
|
|
is shown and embedded until `description` arrives. A reader that sees one
|
|
keeps it.
|
|
|
|
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
|
dict of the fields it found. The shared code calls it for roles not yet read
|
|
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
|
|
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
|
|
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
|
|
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
|
|
and `posted`, the keys whose columns survive a later pull that sends the
|
|
listing's empty value; any other key is left out with a debug log line.
|
|
`title`, `url`, `department` and `salary_text` cannot be filled from
|
|
`detail`: the listing rewrites them every pull. A role not read in detail
|
|
on a pull is sent with an empty description, so the one read earlier stays.
|
|
A 404 or a 410 counts as read; any other failure is logged and retried on a
|
|
later pull; a 429 ends the detail reads of that board for the pull.
|
|
A role read in detail on an earlier pull is sent without the
|
|
listing's place and remote, so what its own page said about them stays.
|
|
Completeness is decided by the listing alone.
|
|
|
|
**robots.txt.** Decided when the reader is built, not checked at run time.
|
|
Whoever builds a reader reads the host's robots.txt for the paths the reader
|
|
requests and reports what it disallows, and the owner decides whether the
|
|
host is read anyway. `robots` records that decision with its date, e.g.
|
|
`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is
|
|
disallowed: owner, 2026-09-30"`. The runtime check in `ats.py`, which
|
|
`robots_page` and `ROBOTS_ANY_CASE` feed, predates this rule and goes with
|
|
the runner; a new reader defines neither. Until then it reads
|
|
`https://<host>/robots.txt` at most once a day per board and fails the board
|
|
when the page is disallowed for `*` or the app's token.
|
|
|
|
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
|
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
|
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
|
|
name is probed as, `(guess,)` without it. A host without `probe_url` is never
|
|
probed, only found by a link.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import pkgutil
|
|
import re
|
|
from collections.abc import Iterable
|
|
from types import ModuleType
|
|
|
|
from . import _contract
|
|
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
|
|
|
|
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
|
|
"Place", "Reads", "Row", "load", "problems", "scope", "undeclared"]
|
|
|
|
#: The rank of a module that states none: after the eight first hosts, whose
|
|
#: ranks (10 to 80) keep the order they had before there were modules.
|
|
DEFAULT_RANK = 100
|
|
|
|
#: What every board module defines.
|
|
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
|
|
|
#: What a module of `scope="search"` defines.
|
|
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed")
|
|
|
|
#: The functions a module may define; each must be callable if present.
|
|
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
|
"fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok",
|
|
"probe_slugs")
|
|
|
|
|
|
def scope(module: ModuleType) -> str:
|
|
"""`board` or `search`; `board` for a module without `READS`."""
|
|
reads = getattr(module, "READS", None)
|
|
return reads.scope if isinstance(reads, Reads) else "board"
|
|
|
|
|
|
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
|
"""What `module` lacks or gets wrong against the contract, one phrase
|
|
each; empty for a module that keeps it. `name` is the module's own name
|
|
in the package, which `NAME` must equal. `READS` is checked when the
|
|
module declares one (`_contract.problems`)."""
|
|
required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED
|
|
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
|
|
if name is not None and getattr(module, "NAME", name) != name:
|
|
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
|
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
|
|
out.append("LABEL is not a non-empty string")
|
|
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
|
out.append("KEEP_CASE is not a bool")
|
|
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
|
|
out.append("ROBOTS_ANY_CASE is not a bool")
|
|
rank = getattr(module, "RANK", DEFAULT_RANK)
|
|
if not isinstance(rank, int) or isinstance(rank, bool):
|
|
out.append("RANK is not an int")
|
|
patterns = getattr(module, "PATTERNS", ())
|
|
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
|
|
out.append("PATTERNS is not a tuple of strings")
|
|
else:
|
|
for pattern in patterns:
|
|
try:
|
|
re.compile(pattern)
|
|
except (TypeError, re.error) as exc:
|
|
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
|
|
out += [f"{attr} is not callable" for attr in FUNCTIONS
|
|
if hasattr(module, attr) and not callable(getattr(module, attr))]
|
|
if hasattr(module, "page"):
|
|
if not hasattr(module, "rows"):
|
|
out.append("page without rows")
|
|
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
|
|
out.append("neither feed_url and parse nor page and rows")
|
|
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
|
|
out.append("probe_ok without probe_url")
|
|
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
|
|
out.append("probe_slugs without probe_url")
|
|
return out + _contract.problems(module)
|
|
|
|
|
|
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
|
|
"""`NAME -> module` for every module of `package` (found on `path`) whose
|
|
name does not start with `_`, in order of `(RANK, NAME)`. Raises
|
|
ImportError naming the first module that breaks the contract."""
|
|
modules: list[ModuleType] = []
|
|
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
|
|
if info.name.startswith("_"):
|
|
continue
|
|
module = importlib.import_module(f"{package}.{info.name}")
|
|
if found := problems(module, info.name):
|
|
raise ImportError(f"{package}.{info.name} breaks the reader contract: "
|
|
+ "; ".join(found))
|
|
modules.append(module)
|
|
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
|
|
return {module.NAME: module for module in modules}
|
|
|
|
|
|
def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]:
|
|
"""The names of the modules (of `READERS` by default) without `READS`,
|
|
read with the older signatures until they declare."""
|
|
return [name for name, module in (READERS if modules is None else modules).items()
|
|
if not hasattr(module, "READS")]
|
|
|
|
|
|
#: name -> module for every reader, in order of `(RANK, NAME)`.
|
|
READERS: dict[str, ModuleType] = load(__path__, __name__)
|
|
|
|
#: host -> its module: the readers of `scope="board"` and the undeclared.
|
|
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
|
|
if scope(module) == "board"}
|
|
|
|
#: provider -> its module: the readers of `scope="search"`.
|
|
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
|
|
if scope(module) == "search"}
|