Files
jobboard/plugins/readers/__init__.py
T
WolfandClaude Fable 5.1 2349fc424b Require READS, search params and a dated robots record; record robots.txt per requested path
Every reader declares, so undeclared(), the scope() fallback, robots_page and
ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may
carry company, places gains none. Each robots record names the paths the
reader requests, from robots.txt read again on 2026-09-30. Platsbanken states
remote, HR Manager states no place.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-30 15:51:13 +02:00

335 lines
18 KiB
Python

"""The readers: one module per source of roles, a host's boards or a search.
A reader registers by existing: every module in this package whose name does
not start with `_` is loaded when the package is imported, checked against
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
`HOSTS` holds those of `scope="board"` (and every module without `READS`),
`PROVIDERS` those of `scope="search"`. A module that breaks the contract
fails the import with an ImportError naming the module and every problem,
rather than a pull that fails board by board. `plugins/ats.py` builds its
allow-list, feeds, pages, the Sources page's host select and its
description from `HOSTS`, and `app/jobs/discover.py` its patterns and
probes. The package sits beside `ats.py` rather than being `plugins/ats/`,
since `desk.sources.registry` loads `plugins/*.py` by path and the tests
patch names on the `ats` module. The types are in `._contract`.
A module is named after its source (`plugins/readers/workday.py`) and
imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It
holds:
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
LABEL = "Workday" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int; lower comes first in the select and the probes
KEEP_CASE = False # optional; True keeps a discovered slug's case
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000,
text="detail", complete="stated",
filters=Filters(country="ids"), places="many",
fields=Fields(posted="listed"),
robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)")
def slug_ok(slug: str) -> bool # strict, see below
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
def careers_url(slug: str) -> str # the page a person opens
and the functions its `READS` names:
def feed_url(slug: str, ask: Ask) -> str # listing="whole": one answer,
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged",
def rows(decoded: Any, slug: str) -> list[Row] # a page a call
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
and, optionally:
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
# read once when `feed_url` is over the cap
def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt)
def probe_url(slug: str) -> str # discovery: 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its
slug is the provider's own (a search reads no board), and it is not
discovered. A module without `READS` is read as a board with the older
signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by
`undeclared()`; the conformance test marks it until it declares.
**Declaration.** `READS` states what the reader does; the runner does it.
`problems()` checks it against the functions: `paged` needs `page` and
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key`
names a config entry, `robots` is not empty, and each function takes the
arguments above. The traits:
- `scope`: `board`, one company's whole list; `search`, many companies,
never a whole list, so nothing a search stops listing is closed by it.
- `listing`: `whole`, one request; `paged`, with `page_size` and `ceiling`,
the offset past which the host answers nothing whatever is asked. A board
above its ceiling is incomplete unless a filter brings it under.
- `text`: `listing`, the list has the full text; `excerpt`, a teaser and no
more; `detail`, one request a role; `none`. `teaser=True` with `detail`
says the list carries a teaser as well.
- `complete`: `stated`, the host gives a total; `end`, an empty page ends
the list; `whole`, the one answer is all; `never`.
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
`one` (one value a request), `many` (a list in one request), `ids` (host
ids looked up with `facets`).
- `places`: `one`, `many`, or `copies` when the host publishes one posting
per place with its own id.
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
posting date; `listed`, a date the host moves; `none`), and `remote`,
`country`, `seniority`, `employment` (`stated` or `none`).
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
and its environment fallbacks. The runner resolves it and passes it in
`Ask.params["key"]`.
- `robots`: the robots.txt decision, see below. Required.
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
far under the last count counts as failed.
- `title`: `provisional` when the listing may lack the title and `detail`
fills it.
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
wants from one series of requests. It sets only dimensions the reader
declared. With `one` it runs a series per value and unions the rows by
`source_id`. With `ids`, `countries` and `places` hold host ids, which the
runner maps from `facets(get, slug)`: `{"country": {"DE": "<host id>"},
"place": {...}}`, called at most once a day per board and kept in state.
`params` holds a search's saved params. `Ask()` is an unfiltered read. A
reader turns `Ask` into its host's query in `feed_url` or `page` and never
filters rows itself.
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
hold 10 to 80, which keeps the order they had before there were modules, so
a new host comes after them unless it says otherwise. The order decides the
select's default and which host wins when discovery's guessed slug exists on
two.
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
which lets a trailing newline through) and refuses anything that could add a
path, a query, a fragment, a port or a user to the URLs built from it.
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
(one DNS label) cover the usual cases; a slug with several parts joins them
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
**Patterns.** The slug is group 1, or, when a pattern has more than one
group, the groups that matched joined by "/". Only group 1 is checked
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
paths are case-sensitive (Workday's site). A pattern without a group names
the host with no slug. A host that `discover.py` names in its own
`_UNREADABLE` table is taken from its module instead as soon as the module
exists.
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
name; `_shared.only(host)` is the usual one. Every request goes through
`get`, which is `_runner.fetch` bound to the target (`_runner.getter`; the
`ats` Source binds it through `ats._get`):
get(url, *, method="GET", payload=None, headers=None) -> bytes
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
default User-Agent and `Accept: application/json, text/xml`, and replace
either where they name it. `get` is https only on port 443, never to an IP
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
most three redirects each checked against `allowed(slug)`, cuts the body at
`_runner.MAX_BYTES` and ends the whole request within `_runner.DEADLINE`. It returns
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
failure, and another exception (`ValueError` among them) for a refused host,
an oversized body or a broken redirect. A module lets these through; the
shared code logs, records and decides.
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
id, to decide completeness), the total the host states or None, and the
cursor of the next page or None after the last. The first call gets cursor
None. A board with a total is complete when as many distinct ids were read;
one without, when the read ended on an empty page. A 429 after the first page
ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
`_runner.PAGE_DELAY` apart, and a board cut there is swept across pulls
(`_runner.read_listing`). A paged module may also have `parse(body, slug)` for one page's body.
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
own id for the role, unique on the board), `title`, `url` (http or https,
checked with `_shared.is_web_url`) and `places`; `location` (the host's
one-line place, kept while storage reads it), `posted` (an aware datetime
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
see `desk.sources.strip_html`), `remote` (True, False, a word or None) and
`salary_text`; where the host states pay, `salary_min`, `salary_max`
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a
new field needs no change there. Undeclared modules do not state `places`
yet.
**Places.** A role has a list of places: `places` is `[Place(text=...,
remote=...)]`, one entry per place the host states, in the host's order, in
the host's words. A reader never multiplies a row per place, never drops a
place and never joins places into one `text`. `remote` is True for a remote
place, False for an office, None when unsaid. A host that publishes one
posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role
that keeps each copy's link.
**Helpers.** `._shared` holds what more than one host needs. A module maps
no word to the app's vocabulary: it hands on the host's own words under
`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them
through the host's table in `app/prep/hosts/<name>.py`.
- `place(text, remote=None)` makes a `Place`: remote when the text has the
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
appends a place unless its text is empty or already in `found`, in the
host's order. A host that states remote itself passes it to `place`, or
builds the dict and passes that to `add_place`.
- `EMPLOYMENTS` is a copy of `app.models.EMPLOYMENTS` for the readers'
tests, since a plugin imports nothing from `app`.
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
over as `teaser`, plain text, whether or not the full text comes later. It
is shown and embedded until `description` arrives. A reader that sees one
keeps it.
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
dict of the fields it found. The shared code calls it for roles not yet read
(`ats:detailed:<target>`), at most `_runner.DETAIL_CAP` (200) a board a
pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that
failed (`ats:detailfailed:<target>`). Only `_runner.DETAIL_FIELDS` are taken:
`description`, `location`, `places`, `teaser`, `remote`, `country`,
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the
listing's empty value; any other key is left out with a debug log line.
`title`, `url`, `department` and `salary_text` cannot be filled from
`detail`: the listing rewrites them every pull. A role not read in detail
on a pull is sent with an empty description, so the one read earlier stays.
A 404 or a 410 counts as read; any other failure is logged and retried on a
later pull; a 429 ends the detail reads of that board for the pull.
A role read in detail on an earlier pull is sent without the
listing's place and remote, so what its own page said about them stays.
Completeness is decided by the listing alone.
**robots.txt.** Decided when the reader is built, not checked at run time.
Whoever builds a reader reads the host's robots.txt for the paths the reader
requests and reports what it disallows, and the owner decides whether the
host is read anyway. `robots` records that decision with its date, e.g.
`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is
disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time;
`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are
read by nothing and a new reader defines neither.
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
the slug is a board; `probe_ok(body)` narrows that when a host answers an
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
name is probed as, `(guess,)` without it. A host without `probe_url` is never
probed, only found by a link.
"""
from __future__ import annotations
import importlib
import pkgutil
import re
from collections.abc import Iterable
from types import ModuleType
from . import _contract
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
"Place", "Reads", "Row", "load", "problems"]
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
DEFAULT_RANK = 100
#: What every board module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: What a module of `scope="search"` defines: `PARAMS` and `clean_params` are
#: what `app/jobs/sources.search_params` stores a search's params with.
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed", "PARAMS", "clean_params")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "detail", "facets", "clean_params", "probe_url", "probe_ok",
"probe_slugs")
def _search(module: ModuleType) -> bool:
reads = getattr(module, "READS", None)
return isinstance(reads, Reads) and reads.scope == "search"
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal. A module without `READS` is
held to a board's names and fails on the missing declaration; the
declaration itself is `_contract.problems`."""
required = REQUIRED_SEARCH if _search(module) else REQUIRED
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
patterns = getattr(module, "PATTERNS", ())
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
out.append("PATTERNS is not a tuple of strings")
else:
for pattern in patterns:
try:
re.compile(pattern)
except (TypeError, re.error) as exc:
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
out += [f"{attr} is not callable" for attr in FUNCTIONS
if hasattr(module, attr) and not callable(getattr(module, attr))]
if hasattr(module, "page"):
if not hasattr(module, "rows"):
out.append("page without rows")
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
out.append("neither feed_url and parse nor page and rows")
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
out.append("probe_ok without probe_url")
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
out.append("probe_slugs without probe_url")
return out + _contract.problems(module)
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
"""`NAME -> module` for every module of `package` (found on `path`) whose
name does not start with `_`, in order of `(RANK, NAME)`. Raises
ImportError naming the first module that breaks the contract."""
modules: list[ModuleType] = []
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{package}.{info.name}")
if found := problems(module, info.name):
raise ImportError(f"{package}.{info.name} breaks the reader contract: "
+ "; ".join(found))
modules.append(module)
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
return {module.NAME: module for module in modules}
#: name -> module for every reader, in order of `(RANK, NAME)`.
READERS: dict[str, ModuleType] = load(__path__, __name__)
#: host -> its module: the readers of `scope="board"`.
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if module.READS.scope == "board"}
#: provider -> its module: the readers of `scope="search"`.
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if module.READS.scope == "search"}