"""The readers: one module per source of roles, a host's boards or a search. A reader registers by existing: every module in this package whose name does not start with `_` is loaded when the package is imported, checked against the contract below, and listed in `READERS` in order of `(RANK, NAME)`; `HOSTS` holds those of `scope="board"`, `PROVIDERS` those of `scope="search"`. A module that breaks the contract, or declares no `READS`, fails the import with an ImportError naming the module and every problem, rather than a pull that fails board by board. `plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host select and its description from `HOSTS`, and `app/jobs/discover.py` its patterns and probes. The package sits beside `ats.py` rather than being `plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and the tests patch names on the `ats` module. The types are in `._contract`. A module is named after its source (`plugins/readers/workday.py`) and imports `desk`, `._contract`, `._shared` and `._countries` only (besides the standard library, `httpx` and `pydantic`), never `plugins.ats` or another reader; `tests/test_architecture.py` holds that. A board's module holds, as Workday's does: NAME = "workday" # the host of a target `workday:`; equals the module's name LABEL = "Workday" # what the Sources page's select shows PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE RANK = 100 # optional int; lower comes first in the select and the probes KEEP_CASE = True # optional; True keeps a discovered slug's case READS = Reads( scope="board", listing="paged", page_size=20, ceiling=2000, text="detail", complete="stated", places="many", filters=Filters(country="ids", remote="ids", query="one"), fields=Fields(posted="trusted", remote="stated", employment="stated"), robots=("read: robots.txt allows // and disallows /refreshFacet/ and has no rule " "on /wday/cxs///, the listing's POST .../jobs and each role's " "JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"), ) def slug_ok(slug: str) -> bool # strict, see below def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach def careers_url(slug: str) -> str # the page a person opens and the functions its `READS` names: def feed_url(slug: str, ask: Ask) -> str # listing="whole": one answer, def parse(body: bytes, slug: str) -> list[Row] # read with `parse` def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged", def rows(decoded: Any, slug: str) -> list[Row] # a page a call def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail" def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter and, optionally: def fallback_url(slug: str) -> str # listing="whole": a smaller feed, # read once when `feed_url` is over the cap def probe_url(slug: str) -> str # discovery: 2xx only for a board def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as **Search providers.** A module of `scope="search"` holds `NAME`, `LABEL`, `allowed`, `READS`, the functions its listing needs, and: PARAMS = {"q": "query", "area": "place"} # saved param -> the filter dimension it fills def clean_params(params: dict[str, Any]) -> dict[str, Any] `app/jobs/sources.search_params` keeps the params of `PARAMS` whose dimension `READS.filters` declares and stores what `clean_params` returns; `clean_params` raises ValueError, with a reason a person can read, for a value the provider would not accept. The runner hands the saved params to `page` in `Ask.params`. A search lists many companies, so its rows carry `company`, the employer's name, which `plugins/search.py` stores. Its slug is the provider's own (a search reads no board), and it has no patterns, probes or careers page: it is never discovered. **Declaration.** `READS` states what the reader does; the runner does it. `problems()` checks it against the functions: `paged` needs `page` and `rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`, `text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs `facets`, `scope="search"` needs `complete="never"` or `"stated"` and `PARAMS` naming filter dimensions, a `key` names a config entry, `robots` is not empty and carries a date, and each function takes the arguments above. `tests/test_readers_contract.py` checks it against the reader's fixtures: each declared field shows on some row, each field declared `none` on none. The traits: - `scope`: `board`, one company's whole list; `search`, many companies, never a whole list, so nothing a search stops listing is closed by it. - `listing`: `whole`, one request; `paged`, with `page_size` and `ceiling`, the offset past which the host answers nothing whatever is asked. A board above its ceiling is incomplete unless a filter brings it under. - `text`: `listing`, the list has the full text; `excerpt`, a teaser and no more; `detail`, one request a role; `none`. `teaser=True` with `detail` says the list carries a teaser as well. - `complete`: `stated`, the host gives a total; `end`, an empty page ends the list; `whole`, the one answer is all; `never`. - `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`, `one` (one value a request), `many` (a list in one request), `ids` (host ids looked up with `facets`). - `places`: `one`, `many`, `copies` when the host publishes one posting per place with its own id, or `none` when the host states no place. - `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the posting date; `listed`, a date the host moves; `none`), and `remote`, `country`, `seniority`, `employment` (`stated` or `none`). A field may come from the listing or from `detail`: BambooHR, Gem and Rippling date a role only on its own page. - `key`: `Key(name, fields, env)`, the config entry `sources.search.` and its environment fallbacks. The runner resolves it and passes it in `Ask.params["key"]`. - `robots`: the robots.txt decision, see below. Required. - `empty`: `suspect` when an unknown slug answers an empty list, so a read far under the last count counts as failed. - `title`: `provisional` when the listing may lack the title and `detail` fills it. Nothing reads it yet: `_runner.DETAIL_FIELDS` has no `title`. **Ask.** `Ask(countries, places, remote, query, params)` is what the runner wants from one series of requests. It sets only dimensions the reader declared. With `one` it runs a series per value and unions the rows by `source_id`. With `ids`, `countries` and `places` hold host ids, which the runner maps from `facets(get, slug)`: `{"country": {"DE": ""}, "place": {...}}`, called at most once a day per board and kept in state. `params` holds a search's saved params. `Ask()` is an unfiltered read. A reader turns `Ask` into its host's query in `feed_url` or `page` and never filters rows itself. **Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts hold 10 to 80, which keeps the order they had before there were modules, so a new host comes after them unless it says otherwise. The order decides the select's default and which host wins when discovery's guessed slug exists on two. **Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`, which lets a trailing newline through) and refuses anything that could add a path, a query, a fragment, a port or a user to the URLs built from it. `_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok` (one DNS label) cover the usual cases; a slug with several parts joins them with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`). **Patterns.** The slug is group 1, or, when a pattern has more than one group, the groups that matched joined by "/". Only group 1 is checked against discovery's words that name no board (`api`, `jobs`, `embed`, ...). The slug is lowercased unless `KEEP_CASE` is True: set it when the host's paths are case-sensitive (Workday's site). A pattern without a group names the host with no slug. A host that `discover.py` names in its own `_UNREADABLE` table loses its entry there when its module is added. **Fetching.** `allowed(slug)` returns a predicate over a lowercased host name; `_shared.only(host)` is the usual one. Every request goes through `get`, which is `_runner.fetch` bound to the target (`_runner.getter`; the `ats` Source binds it through `ats._get`): get(url, *, method="GET", payload=None, headers=None) -> bytes A POST sends `payload` as JSON (`{}` when None); `headers` are added to the default User-Agent and `Accept: application/json, text/xml`, and replace either where they name it. `get` is https only on port 443, never to an IP address or a local name (`localhost`, `.local`, `.internal`, ...), follows at most three redirects each checked against `allowed(slug)`, cuts the body at `_runner.MAX_BYTES` and ends the whole request within `_runner.DEADLINE`. It returns the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read `exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection failure, and another exception (`ValueError` among them) for a refused host, an oversized body or a broken redirect. A module lets these through; the shared code logs, records and decides. **Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page handed to `rows`, its raw results as dicts with an `id` (counted, by distinct id, to decide completeness), the total the host states or None, and the cursor of the next page or None after the last. The first call gets cursor None. A board with a total is complete when as many distinct ids were read; one without, when the read ended on an empty page. A 429 after the first page ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most, `_runner.PAGE_DELAY` apart, and a board cut there is swept across pulls (`_runner.read_listing`). A paged module may also have `parse(body, slug)` for one page's body. **Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's own id for the role, unique on the board), `title`, `url` (http or https, checked with `_shared.is_web_url`) and `places`; `location` (the host's one-line place, kept while storage reads it), `posted` (an aware datetime or None, see `_shared.parse_dt`), `department`, `description` (plain text, see `desk.sources.strip_html`), `remote` (True, False, a word or None) and `salary_text`; where the host states pay, `salary_min`, `salary_max` (yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`; where it states them, `country` (ISO 3166 alpha-2), `seniority` and `employment`; `teaser`; and, on a search row, `company`. The runner keeps every stated key in `raw`, so a new field needs no change there. **Places.** A role has a list of places: `places` is `[Place(text=..., remote=...)]`, one entry per place the host states, in the host's order, in the host's words. A reader never multiplies a row per place, never drops a place and never joins places into one `text`. `remote` is True for a remote place, False for an office, None when unsaid. A host that publishes one posting per place, each with its own id, declares `places="copies"`: each copy is a row with its one place, and `dupes.py` folds them into one role that keeps each copy's link. A host that states no place declares `places="none"` and hands over `places == []`. **Helpers.** `._shared` holds what more than one host needs, `._countries` the country names hosts write. A module maps no word to the app's vocabulary: it hands on the host's own words under `seniority`, `employment`, `remote` and `country`, and `app.prep` maps them through the host's table in `app/prep/hosts/.py`. - `place(text, remote=None)` makes a `Place`: remote when the text has the word `remote`, otherwise as `remote` says. `add_place(found, entry)` appends a place unless its text is empty or already in `found`, in the host's order. A host that states remote itself passes it to `place`, or builds the dict and passes that to `add_place`. - `EMPLOYMENTS` is a copy of `app.models.EMPLOYMENTS` for the readers' tests, since a plugin imports nothing from `app`. **Teaser.** A short text the list carries (an excerpt, a summary) is handed over as `teaser`, plain text, whether or not the full text comes later. It is shown and embedded until `description` arrives. A reader that sees one keeps it. **Detail.** `detail(get, slug, row)` reads one role's own page and returns a dict of the fields it found. The pull reads listings only; the `enrich` stage (`app/jobs/enrich.py`) calls `detail` after scoring, so the requests go where the ranking is: the open roles without text that an agent ranks among its best `enrich.top` at `enrich.min_score` or above, best first, at most `enrich.per_board` from one board and `enrich.per_run` in all, one batch per board through `ats.read_board_details` and `_runner.read_details`, `_runner.PAGE_DELAY` apart, the roles never tried before those whose read failed. Only `_runner.DETAIL_FIELDS` are taken: `description`, `location`, `places`, `teaser`, `remote`, `country`, `employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the listing's empty value; any other key is left out with a debug log line, `seniority` among them. `title`, `url`, `department` and `salary_text` cannot be filled from `detail`: the listing rewrites them every pull. A 404 or a 410 counts as read; any other failure is logged and retried on a later run; a read that gave no text is retried after `enrich.retry_days`; a 429 ends the reads of that board for the run. A role read in detail is sent by later pulls without the listing's place and remote, so what its own page said about them stays. Completeness is decided by the listing alone. **robots.txt.** Decided when the reader is built, not checked at run time. Whoever builds a reader reads the robots.txt of every host the reader requests, for the paths it requests (the API's, not the career page a person opens), and the owner decides whether the host is read anyway. `robots` records what the file says of those paths, the date it was read and the decision, e.g. `"read: robots.txt has no rule on /wday/cxs/ (checked 2026-09-30)"` or `"read though /api/ is disallowed: owner, 2026-09-30"`; a record without a date fails the contract. Nothing reads robots.txt at run time. **Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when the slug is a board; `probe_ok(body)` narrows that when a host answers an empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed name is probed as, `(guess,)` without it. A host without `probe_url` is never probed, only found by a link. **Adding a reader.** A new host or provider touches: - its module here; - `app/prep/hosts/.py` when it states seniority or employment in its own words (the contract test requires the table; a `REMOTE` table when its remote words are not the app's); - its fixtures under `tests/fixtures/ats/` (`tests/fixtures/search/` for a provider), named as `tests/test_readers_contract.py` describes, and `tests/test_ats_.py` and `tests/test_prep_hosts_.py`; - its entry in `discover._UNREADABLE`, deleted, where discovery named the host before; - a section in `plugins/README.md`, and a row in `docs/filter-survey.md` when it filters at the source. Nothing else: `HOSTS` and `PROVIDERS`, the Sources page's select, discovery's patterns and probes, and the reach a board is read with are derived from the module. """ from __future__ import annotations import importlib import pkgutil import re from collections.abc import Iterable from types import ModuleType from . import _contract from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row __all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key", "Place", "Reads", "Row", "load", "problems"] #: The rank of a module that states none: after the eight first hosts, whose #: ranks (10 to 80) keep the order they had before there were modules. DEFAULT_RANK = 100 #: What every board module defines. REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url") #: What a module of `scope="search"` defines: `PARAMS` and `clean_params` are #: what `app/jobs/sources.search_params` stores a search's params with. REQUIRED_SEARCH = ("NAME", "LABEL", "allowed", "PARAMS", "clean_params") #: The functions a module may define; each must be callable if present. FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows", "fallback_url", "detail", "facets", "clean_params", "probe_url", "probe_ok", "probe_slugs") def _search(module: ModuleType) -> bool: reads = getattr(module, "READS", None) return isinstance(reads, Reads) and reads.scope == "search" def problems(module: ModuleType, name: str | None = None) -> list[str]: """What `module` lacks or gets wrong against the contract, one phrase each; empty for a module that keeps it. `name` is the module's own name in the package, which `NAME` must equal. A module without `READS` is held to a board's names and fails on the missing declaration; the declaration itself is `_contract.problems`.""" required = REQUIRED_SEARCH if _search(module) else REQUIRED out = [f"no {attr}" for attr in required if not hasattr(module, attr)] if name is not None and getattr(module, "NAME", name) != name: out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}") if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL): out.append("LABEL is not a non-empty string") if not isinstance(getattr(module, "KEEP_CASE", False), bool): out.append("KEEP_CASE is not a bool") rank = getattr(module, "RANK", DEFAULT_RANK) if not isinstance(rank, int) or isinstance(rank, bool): out.append("RANK is not an int") patterns = getattr(module, "PATTERNS", ()) if isinstance(patterns, str) or not isinstance(patterns, Iterable): out.append("PATTERNS is not a tuple of strings") else: for pattern in patterns: try: re.compile(pattern) except (TypeError, re.error) as exc: out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})") out += [f"{attr} is not callable" for attr in FUNCTIONS if hasattr(module, attr) and not callable(getattr(module, attr))] if hasattr(module, "page"): if not hasattr(module, "rows"): out.append("page without rows") elif not (hasattr(module, "feed_url") and hasattr(module, "parse")): out.append("neither feed_url and parse nor page and rows") if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"): out.append("probe_ok without probe_url") if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"): out.append("probe_slugs without probe_url") return out + _contract.problems(module) def load(path: Iterable[str], package: str) -> dict[str, ModuleType]: """`NAME -> module` for every module of `package` (found on `path`) whose name does not start with `_`, in order of `(RANK, NAME)`. Raises ImportError naming the first module that breaks the contract.""" modules: list[ModuleType] = [] for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name): if info.name.startswith("_"): continue module = importlib.import_module(f"{package}.{info.name}") if found := problems(module, info.name): raise ImportError(f"{package}.{info.name} breaks the reader contract: " + "; ".join(found)) modules.append(module) modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME)) return {module.NAME: module for module in modules} #: name -> module for every reader, in order of `(RANK, NAME)`. READERS: dict[str, ModuleType] = load(__path__, __name__) #: host -> its module: the readers of `scope="board"`. HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items() if module.READS.scope == "board"} #: provider -> its module: the readers of `scope="search"`. PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items() if module.READS.scope == "search"}