Files
jobboard/plugins/readers/__init__.py
T

379 lines
21 KiB
Python

"""The readers: one module per source of roles, a host's boards or a search.
A reader registers by existing: every module in this package whose name does
not start with `_` is loaded when the package is imported, checked against
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
`HOSTS` holds those of `scope="board"`, `PROVIDERS` those of
`scope="search"`. A module that breaks the contract, or declares no `READS`,
fails the import with an ImportError naming the module and every problem,
rather than a pull that fails board by board. `plugins/ats.py` builds its
allow-list, feeds, pages, the Sources page's host select and its
description from `HOSTS`, and `app/jobs/discover.py` its patterns and
probes. The package sits beside `ats.py` rather than being `plugins/ats/`,
since `desk.sources.registry` loads `plugins/*.py` by path and the tests
patch names on the `ats` module. The types are in `._contract`.
A module is named after its source (`plugins/readers/workday.py`) and
imports `desk`, `._contract`, `._shared` and `._countries` only (besides the
standard library, `httpx` and `pydantic`), never `plugins.ats` or another
reader; `tests/test_architecture.py` holds that. A board's module holds, as
Workday's does:
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
LABEL = "Workday" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int; lower comes first in the select and the probes
KEEP_CASE = True # optional; True keeps a discovered slug's case
READS = Reads(
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
complete="stated", places="many",
filters=Filters(country="ids", remote="ids", query="one"),
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots=("read: robots.txt allows /<Site>/ and disallows /refreshFacet/ and has no rule "
"on /wday/cxs/<tenant>/<site>/, the listing's POST .../jobs and each role's "
"JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"),
)
def slug_ok(slug: str) -> bool # strict, see below
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
def careers_url(slug: str) -> str # the page a person opens
and the functions its `READS` names:
def feed_url(slug: str, ask: Ask) -> str # listing="whole": one answer,
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged",
def rows(decoded: Any, slug: str) -> list[Row] # a page a call
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
and, optionally:
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
# read once when `feed_url` is over the cap
def probe_url(slug: str) -> str # discovery: 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
**Search providers.** A module of `scope="search"` holds `NAME`, `LABEL`,
`allowed`, `READS`, the functions its listing needs, and:
PARAMS = {"q": "query", "area": "place"} # saved param -> the filter dimension it fills
def clean_params(params: dict[str, Any]) -> dict[str, Any]
`app/jobs/sources.search_params` keeps the params of `PARAMS` whose
dimension `READS.filters` declares and stores what `clean_params` returns;
`clean_params` raises ValueError, with a reason a person can read, for a
value the provider would not accept. The runner hands the saved params to
`page` in `Ask.params`. A search lists many companies, so its rows carry
`company`, the employer's name, which `plugins/search.py` stores. Its slug
is the provider's own (a search reads no board), and it has no patterns,
probes or careers page: it is never discovered.
**Declaration.** `READS` states what the reader does; the runner does it.
`problems()` checks it against the functions: `paged` needs `page` and
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
`facets`, `scope="search"` needs `complete="never"` or `"stated"` and
`PARAMS` naming filter dimensions, a `key` names a config entry, `robots`
is not empty and carries a date, and each function takes the arguments
above. `tests/test_readers_contract.py` checks it against the reader's
fixtures: each declared field shows on some row, each field declared `none`
on none. The traits:
- `scope`: `board`, one company's whole list; `search`, many companies,
never a whole list, so nothing a search stops listing is closed by it.
- `listing`: `whole`, one request; `paged`, with `page_size` and `ceiling`,
the offset past which the host answers nothing whatever is asked. A board
above its ceiling is incomplete unless a filter brings it under.
- `text`: `listing`, the list has the full text; `excerpt`, a teaser and no
more; `detail`, one request a role; `none`. `teaser=True` with `detail`
says the list carries a teaser as well.
- `complete`: `stated`, the host gives a total; `end`, an empty page ends
the list; `whole`, the one answer is all; `never`.
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
`one` (one value a request), `many` (a list in one request), `ids` (host
ids looked up with `facets`).
- `places`: `one`, `many`, `copies` when the host publishes one posting
per place with its own id, or `none` when the host states no place.
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
posting date; `listed`, a date the host moves; `none`), and `remote`,
`country`, `seniority`, `employment` (`stated` or `none`). A field may
come from the listing or from `detail`: BambooHR, Gem and Rippling date a
role only on its own page.
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
and its environment fallbacks. The runner resolves it and passes it in
`Ask.params["key"]`.
- `robots`: the robots.txt decision, see below. Required.
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
far under the last count counts as failed.
- `title`: `provisional` when the listing may lack the title and `detail`
fills it. Nothing reads it yet: `_runner.DETAIL_FIELDS` has no `title`.
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
wants from one series of requests. It sets only dimensions the reader
declared. With `one` it runs a series per value and unions the rows by
`source_id`. With `ids`, `countries` and `places` hold host ids, which the
runner maps from `facets(get, slug)`: `{"country": {"DE": "<host id>"},
"place": {...}}`, called at most once a day per board and kept in state.
`params` holds a search's saved params. `Ask()` is an unfiltered read. A
reader turns `Ask` into its host's query in `feed_url` or `page` and never
filters rows itself.
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
hold 10 to 80, which keeps the order they had before there were modules, so
a new host comes after them unless it says otherwise. The order decides the
select's default and which host wins when discovery's guessed slug exists on
two.
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
which lets a trailing newline through) and refuses anything that could add a
path, a query, a fragment, a port or a user to the URLs built from it.
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
(one DNS label) cover the usual cases; a slug with several parts joins them
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
**Patterns.** The slug is group 1, or, when a pattern has more than one
group, the groups that matched joined by "/". Only group 1 is checked
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
paths are case-sensitive (Workday's site). A pattern without a group names
the host with no slug. A host that `discover.py` names in its own
`_UNREADABLE` table loses its entry there when its module is added.
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
name; `_shared.only(host)` is the usual one. Every request goes through
`get`, which is `_runner.fetch` bound to the target (`_runner.getter`; the
`ats` Source binds it through `ats._get`):
get(url, *, method="GET", payload=None, headers=None) -> bytes
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
default User-Agent and `Accept: application/json, text/xml`, and replace
either where they name it. `get` is https only on port 443, never to an IP
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
most three redirects each checked against `allowed(slug)`, cuts the body at
`_runner.MAX_BYTES` and ends the whole request within `_runner.DEADLINE`. It returns
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
failure, and another exception (`ValueError` among them) for a refused host,
an oversized body or a broken redirect. A module lets these through; the
shared code logs, records and decides.
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
id, to decide completeness), the total the host states or None, and the
cursor of the next page or None after the last. The first call gets cursor
None, and a later one may get its cursor back from JSON, a tuple as a list,
when a sweep continues. A board with a total is complete when as many
distinct ids were read; one without, when the read ended on an empty page.
A reader that sees a page come again hands it over with no results and a
cursor that is not None, which ends the read incomplete; None there would
read as the last page. A 429 after the first page
ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
`_runner.PAGE_DELAY` apart, and a board cut there is swept across pulls
(`_runner.read_listing`). A paged module may also have `parse(body, slug)` for one page's body.
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
own id for the role, unique on the board), `title`, `url` (http or https,
checked with `_shared.is_web_url`) and `places`; `location` (the host's
one-line place, kept while storage reads it), `posted` (an aware datetime
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
see `_shared.html_text`), `remote` (True, False, a word or None) and
`salary_text`; where the host states pay, `salary_min`, `salary_max`
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
every stated key in `raw`, so a new field needs no change there.
**Places.** A role has a list of places: `places` is `[Place(text=...,
remote=...)]`, one entry per place the host states, in the host's order, in
the host's words. A reader never multiplies a row per place, never drops a
place and never joins places into one `text`. `remote` is True for a remote
place, False for an office, None when unsaid. A host that publishes one
posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role
that keeps each copy's link. A host that states no place declares
`places="none"` and hands over `places == []`.
**Helpers.** `._shared` holds what more than one host needs, `._countries`
the country names hosts write. A module maps no word to the app's
vocabulary: it hands on the host's own words under `seniority`,
`employment`, `remote` and `country`, and `app.prep` maps them through the
host's table in `app/prep/hosts/<name>.py`.
- `place(text, remote=None)` makes a `Place`: remote when the text has the
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
appends a place unless its text is empty or already in `found`, in the
host's order. A host that states remote itself passes it to `place`, or
builds the dict and passes that to `add_place`.
- `EMPLOYMENTS` is a copy of `app.models.EMPLOYMENTS` for the readers'
tests, since a plugin imports nothing from `app`.
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
over as `teaser`, plain text, whether or not the full text comes later. It
is shown and embedded until `description` arrives. A reader that sees one
keeps it.
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
dict of the fields it found. The pull reads listings only; the `fetch_text`
stage (`app/jobs/fetch_text.py`) calls `detail` after scoring, so the
requests go where the model pass will read: the open roles without text in
the cut of an enabled agent, best first, at most `fetch_text.per_board`
from one board and `fetch_text.per_run` in all, one
batch per board through `ats.read_board_details` and
`_runner.read_details`, `_runner.PAGE_DELAY` apart, the roles never tried
before those whose read failed. Only `_runner.DETAIL_FIELDS` are taken:
`description`, `location`, `places`, `teaser`, `remote`, `country`,
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys
whose columns survive a later pull that sends the listing's empty value;
any other key is left out with a debug log line, `seniority` among them.
`title`, `url`, `department` and `salary_text` cannot be filled from
`detail`: the listing rewrites them every pull. A 404 or a 410 counts as read; any
other failure is logged and retried on a later run; a read that gave no text
is retried after `fetch_text.retry_days`; a 429 ends the reads of that board for
the run. A role read in detail is sent by later pulls without the listing's
place and remote, so what its own page said about them stays. Completeness
is decided by the listing alone.
**robots.txt.** Decided when the reader is built, not checked at run time.
Whoever builds a reader reads the robots.txt of every host the reader
requests, for the paths it requests (the API's, not the career page a
person opens), and the owner decides whether the host is read anyway.
`robots` records what the file says of those paths, the date it was read
and the decision, e.g. `"read: robots.txt has no rule on /wday/cxs/
(checked 2026-09-30)"` or `"read though /api/ is disallowed: owner,
2026-09-30"`; a record without a date fails the contract. Nothing reads
robots.txt at run time.
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
the slug is a board; `probe_ok(body)` narrows that when a host answers an
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
name is probed as, `(guess,)` without it. A host without `probe_url` is never
probed, only found by a link.
**Adding a reader.** A new host or provider touches:
- its module here;
- `app/prep/hosts/<name>.py` when it states seniority or employment in its
own words (the contract test requires the table; a `REMOTE` table when
its remote words are not the app's);
- its fixtures under `tests/fixtures/ats/` (`tests/fixtures/search/` for a
provider), named as `tests/test_readers_contract.py` describes, and
`tests/test_ats_<name>.py` and `tests/test_prep_hosts_<name>.py`;
- its entry in `discover._UNREADABLE`, deleted, where discovery named the
host before;
- a section in `plugins/README.md`, and a row in `docs/filter-survey.md`
when it filters at the source.
Nothing else: `HOSTS` and `PROVIDERS`, the Sources page's select,
discovery's patterns and probes, and the reach a board is read with are
derived from the module.
"""
from __future__ import annotations
import importlib
import pkgutil
import re
from collections.abc import Iterable
from types import ModuleType
from . import _contract
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
"Place", "Reads", "Row", "load", "problems"]
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
DEFAULT_RANK = 100
#: What every board module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: What a module of `scope="search"` defines: `PARAMS` and `clean_params` are
#: what `app/jobs/sources.search_params` stores a search's params with.
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed", "PARAMS", "clean_params")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "detail", "facets", "clean_params", "probe_url", "probe_ok",
"probe_slugs")
def _search(module: ModuleType) -> bool:
reads = getattr(module, "READS", None)
return isinstance(reads, Reads) and reads.scope == "search"
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal. A module without `READS` is
held to a board's names and fails on the missing declaration; the
declaration itself is `_contract.problems`."""
required = REQUIRED_SEARCH if _search(module) else REQUIRED
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
patterns = getattr(module, "PATTERNS", ())
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
out.append("PATTERNS is not a tuple of strings")
else:
for pattern in patterns:
try:
re.compile(pattern)
except (TypeError, re.error) as exc:
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
out += [f"{attr} is not callable" for attr in FUNCTIONS
if hasattr(module, attr) and not callable(getattr(module, attr))]
if hasattr(module, "page"):
if not hasattr(module, "rows"):
out.append("page without rows")
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
out.append("neither feed_url and parse nor page and rows")
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
out.append("probe_ok without probe_url")
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
out.append("probe_slugs without probe_url")
return out + _contract.problems(module)
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
"""`NAME -> module` for every module of `package` (found on `path`) whose
name does not start with `_`, in order of `(RANK, NAME)`. Raises
ImportError naming the first module that breaks the contract."""
modules: list[ModuleType] = []
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{package}.{info.name}")
if found := problems(module, info.name):
raise ImportError(f"{package}.{info.name} breaks the reader contract: "
+ "; ".join(found))
modules.append(module)
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
return {module.NAME: module for module in modules}
#: name -> module for every reader, in order of `(RANK, NAME)`.
READERS: dict[str, ModuleType] = load(__path__, __name__)
#: host -> its module: the readers of `scope="board"`.
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if module.READS.scope == "board"}
#: provider -> its module: the readers of `scope="search"`.
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if module.READS.scope == "search"}