diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py index 5e873fe..6134d17 100644 --- a/plugins/readers/__init__.py +++ b/plugins/readers/__init__.py @@ -3,8 +3,8 @@ A reader registers by existing: every module in this package whose name does not start with `_` is loaded when the package is imported, checked against the contract below, and listed in `READERS` in order of `(RANK, NAME)`; -`HOSTS` holds those of `scope="board"` (and every module without `READS`), -`PROVIDERS` those of `scope="search"`. A module that breaks the contract +`HOSTS` holds those of `scope="board"`, `PROVIDERS` those of +`scope="search"`. A module that breaks the contract, or declares no `READS`, fails the import with an ImportError naming the module and every problem, rather than a pull that fails board by board. `plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host select and its @@ -14,21 +14,26 @@ since `desk.sources.registry` loads `plugins/*.py` by path and the tests patch names on the `ats` module. The types are in `._contract`. A module is named after its source (`plugins/readers/workday.py`) and -imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It -holds: +imports `desk`, `._contract`, `._shared` and `._countries` only (besides the +standard library, `httpx` and `pydantic`), never `plugins.ats` or another +reader; `tests/test_architecture.py` holds that. A board's module holds, as +Workday's does: NAME = "workday" # the host of a target `workday:`; equals the module's name LABEL = "Workday" # what the Sources page's select shows PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE RANK = 100 # optional int; lower comes first in the select and the probes - KEEP_CASE = False # optional; True keeps a discovered slug's case - ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case + KEEP_CASE = True # optional; True keeps a discovered slug's case - READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000, - text="detail", complete="stated", - filters=Filters(country="ids"), places="many", - fields=Fields(posted="listed"), - robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)") + READS = Reads( + scope="board", listing="paged", page_size=20, ceiling=2000, text="detail", + complete="stated", places="many", + filters=Filters(country="ids", remote="ids", query="one"), + fields=Fields(posted="trusted", remote="stated", employment="stated"), + robots=("read: robots.txt allows // and disallows /refreshFacet/ and has no rule " + "on /wday/cxs///, the listing's POST .../jobs and each role's " + "JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"), + ) def slug_ok(slug: str) -> bool # strict, see below def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach @@ -49,24 +54,35 @@ and, optionally: def fallback_url(slug: str) -> str # listing="whole": a smaller feed, # read once when `feed_url` is over the cap - def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt) def probe_url(slug: str) -> str # discovery: 2xx only for a board def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as -A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its -slug is the provider's own (a search reads no board), and it is not -discovered. A module without `READS` is read as a board with the older -signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by -`undeclared()`; the conformance test marks it until it declares. +**Search providers.** A module of `scope="search"` holds `NAME`, `LABEL`, +`allowed`, `READS`, the functions its listing needs, and: + + PARAMS = {"q": "query", "area": "place"} # saved param -> the filter dimension it fills + def clean_params(params: dict[str, Any]) -> dict[str, Any] + +`app/jobs/sources.search_params` keeps the params of `PARAMS` whose +dimension `READS.filters` declares and stores what `clean_params` returns; +`clean_params` raises ValueError, with a reason a person can read, for a +value the provider would not accept. The runner hands the saved params to +`page` in `Ask.params`. A search lists many companies, so its rows carry +`company`, the employer's name, which `plugins/search.py` stores. Its slug +is the provider's own (a search reads no board), and it has no patterns, +probes or careers page: it is never discovered. **Declaration.** `READS` states what the reader does; the runner does it. `problems()` checks it against the functions: `paged` needs `page` and `rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`, `text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs -`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key` -names a config entry, `robots` is not empty, and each function takes the -arguments above. The traits: +`facets`, `scope="search"` needs `complete="never"` or `"stated"` and +`PARAMS` naming filter dimensions, a `key` names a config entry, `robots` +is not empty and carries a date, and each function takes the arguments +above. `tests/test_readers_contract.py` checks it against the reader's +fixtures: each declared field shows on some row, each field declared `none` +on none. The traits: - `scope`: `board`, one company's whole list; `search`, many companies, never a whole list, so nothing a search stops listing is closed by it. @@ -81,11 +97,13 @@ arguments above. The traits: - `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`, `one` (one value a request), `many` (a list in one request), `ids` (host ids looked up with `facets`). -- `places`: `one`, `many`, or `copies` when the host publishes one posting - per place with its own id. +- `places`: `one`, `many`, `copies` when the host publishes one posting + per place with its own id, or `none` when the host states no place. - `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the posting date; `listed`, a date the host moves; `none`), and `remote`, - `country`, `seniority`, `employment` (`stated` or `none`). + `country`, `seniority`, `employment` (`stated` or `none`). A field may + come from the listing or from `detail`: BambooHR, Gem and Rippling date a + role only on its own page. - `key`: `Key(name, fields, env)`, the config entry `sources.search.` and its environment fallbacks. The runner resolves it and passes it in `Ask.params["key"]`. @@ -93,7 +111,7 @@ arguments above. The traits: - `empty`: `suspect` when an unknown slug answers an empty list, so a read far under the last count counts as failed. - `title`: `provisional` when the listing may lack the title and `detail` - fills it. + fills it. Nothing reads it yet: `_runner.DETAIL_FIELDS` has no `title`. **Ask.** `Ask(countries, places, remote, query, params)` is what the runner wants from one series of requests. It sets only dimensions the reader @@ -124,8 +142,7 @@ against discovery's words that name no board (`api`, `jobs`, `embed`, ...). The slug is lowercased unless `KEEP_CASE` is True: set it when the host's paths are case-sensitive (Workday's site). A pattern without a group names the host with no slug. A host that `discover.py` names in its own -`_UNREADABLE` table is taken from its module instead as soon as the module -exists. +`_UNREADABLE` table loses its entry there when its module is added. **Fetching.** `allowed(slug)` returns a predicate over a lowercased host name; `_shared.only(host)` is the usual one. Every request goes through @@ -165,9 +182,8 @@ see `desk.sources.strip_html`), `remote` (True, False, a word or None) and `salary_text`; where the host states pay, `salary_min`, `salary_max` (yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`; where it states them, `country` (ISO 3166 alpha-2), `seniority` and -`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a -new field needs no change there. Undeclared modules do not state `places` -yet. +`employment`; `teaser`; and, on a search row, `company`. The runner keeps +every stated key in `raw`, so a new field needs no change there. **Places.** A role has a list of places: `places` is `[Place(text=..., remote=...)]`, one entry per place the host states, in the host's order, in @@ -176,12 +192,14 @@ place and never joins places into one `text`. `remote` is True for a remote place, False for an office, None when unsaid. A host that publishes one posting per place, each with its own id, declares `places="copies"`: each copy is a row with its one place, and `dupes.py` folds them into one role -that keeps each copy's link. +that keeps each copy's link. A host that states no place declares +`places="none"` and hands over `places == []`. -**Helpers.** `._shared` holds what more than one host needs. A module maps -no word to the app's vocabulary: it hands on the host's own words under -`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them -through the host's table in `app/prep/hosts/.py`. +**Helpers.** `._shared` holds what more than one host needs, `._countries` +the country names hosts write. A module maps no word to the app's +vocabulary: it hands on the host's own words under `seniority`, +`employment`, `remote` and `country`, and `app.prep` maps them through the +host's table in `app/prep/hosts/.py`. - `place(text, remote=None)` makes a `Place`: remote when the text has the word `remote`, otherwise as `remote` says. `add_place(found, entry)` @@ -197,38 +215,60 @@ is shown and embedded until `description` arrives. A reader that sees one keeps it. **Detail.** `detail(get, slug, row)` reads one role's own page and returns a -dict of the fields it found. The shared code calls it for roles not yet read -(`ats:detailed:`), at most `_runner.DETAIL_CAP` (200) a board a -pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that -failed (`ats:detailfailed:`). Only `_runner.DETAIL_FIELDS` are taken: +dict of the fields it found. The pull reads listings only; the `enrich` +stage (`app/jobs/enrich.py`) calls `detail` after scoring, so the requests +go where the ranking is: the open roles without text that an agent ranks +among its best `enrich.top` at `enrich.min_score` or above, best first, at +most `enrich.per_board` from one board and `enrich.per_run` in all, one +batch per board through `ats.read_board_details` and +`_runner.read_details`, `_runner.PAGE_DELAY` apart, the roles never tried +before those whose read failed. Only `_runner.DETAIL_FIELDS` are taken: `description`, `location`, `places`, `teaser`, `remote`, `country`, -`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the -listing's empty value; any other key is left out with a debug log line. +`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys +whose columns survive a later pull that sends the listing's empty value; +any other key is left out with a debug log line, `seniority` among them. `title`, `url`, `department` and `salary_text` cannot be filled from -`detail`: the listing rewrites them every pull. A role not read in detail -on a pull is sent with an empty description, so the one read earlier stays. -A 404 or a 410 counts as read; any other failure is logged and retried on a -later pull; a 429 ends the detail reads of that board for the pull. -A role read in detail on an earlier pull is sent without the -listing's place and remote, so what its own page said about them stays. -Completeness is decided by the listing alone. +`detail`: the listing rewrites them every pull. A 404 or a 410 counts as read; any +other failure is logged and retried on a later run; a read that gave no text +is retried after `enrich.retry_days`; a 429 ends the reads of that board for +the run. A role read in detail is sent by later pulls without the listing's +place and remote, so what its own page said about them stays. Completeness +is decided by the listing alone. **robots.txt.** Decided when the reader is built, not checked at run time. -Whoever builds a reader reads the host's robots.txt for the paths the reader -requests and reports what it disallows, and the owner decides whether the -host is read anyway. `robots` records that decision with its date, e.g. -`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is -disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time; -`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are -read by nothing and a new reader defines neither. +Whoever builds a reader reads the robots.txt of every host the reader +requests, for the paths it requests (the API's, not the career page a +person opens), and the owner decides whether the host is read anyway. +`robots` records what the file says of those paths, the date it was read +and the decision, e.g. `"read: robots.txt has no rule on /wday/cxs/ +(checked 2026-09-30)"` or `"read though /api/ is disallowed: owner, +2026-09-30"`; a record without a date fails the contract. Nothing reads +robots.txt at run time. **Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when the slug is a board; `probe_ok(body)` narrows that when a host answers an empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed name is probed as, `(guess,)` without it. A host without `probe_url` is never probed, only found by a link. -""" +**Adding a reader.** A new host or provider touches: + +- its module here; +- `app/prep/hosts/.py` when it states seniority or employment in its + own words (the contract test requires the table; a `REMOTE` table when + its remote words are not the app's); +- its fixtures under `tests/fixtures/ats/` (`tests/fixtures/search/` for a + provider), named as `tests/test_readers_contract.py` describes, and + `tests/test_ats_.py` and `tests/test_prep_hosts_.py`; +- its entry in `discover._UNREADABLE`, deleted, where discovery named the + host before; +- a section in `plugins/README.md`, and a row in `docs/filter-survey.md` + when it filters at the source. + +Nothing else: `HOSTS` and `PROVIDERS`, the Sources page's select, +discovery's patterns and probes, and the reach a board is read with are +derived from the module. +""" from __future__ import annotations import importlib diff --git a/tests/test_readers_contract.py b/tests/test_readers_contract.py index 859b060..4d739e6 100644 --- a/tests/test_readers_contract.py +++ b/tests/test_readers_contract.py @@ -32,6 +32,7 @@ already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels). from __future__ import annotations import sys +import textwrap import types from collections.abc import Callable from pathlib import Path @@ -535,6 +536,15 @@ def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None: assert place_problems(rows, "many") == [] +def test_the_contracts_example_is_workdays_declaration() -> None: + doc = readers.__doc__ or "" + start = doc.index(" READS = Reads(") + source = textwrap.dedent(doc[start:doc.index("\n )\n", start) + 6]) + namespace: dict[str, Any] = {"Reads": Reads, "Filters": Filters, "Fields": Fields} + exec(source, namespace) + assert namespace["READS"] == readers.READERS["workday"].READS + + def test_the_contract_types_are_what_the_package_exports() -> None: assert readers.Reads is _contract.Reads and readers.Row is _contract.Row assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}