Correct the contract docstring: detail is read by enrich, the example is Workday's, the touch points of a new reader

Adds the search providers' PARAMS, clean_params and company, and a test
that the example READS is workday.py's.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:54:13 +02:00
1 parent 72252a1c04
commit 2cc83b1d84
2 files changed
+105 -55

No files matched your search

+95 -55
View File
@@ -3,8 +3,8 @@
A reader registers by existing: every module in this package whose name does A reader registers by existing: every module in this package whose name does
not start with `_` is loaded when the package is imported, checked against not start with `_` is loaded when the package is imported, checked against
the contract below, and listed in `READERS` in order of `(RANK, NAME)`; the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
`HOSTS` holds those of `scope="board"` (and every module without `READS`), `HOSTS` holds those of `scope="board"`, `PROVIDERS` those of
`PROVIDERS` those of `scope="search"`. A module that breaks the contract `scope="search"`. A module that breaks the contract, or declares no `READS`,
fails the import with an ImportError naming the module and every problem, fails the import with an ImportError naming the module and every problem,
rather than a pull that fails board by board. `plugins/ats.py` builds its rather than a pull that fails board by board. `plugins/ats.py` builds its
allow-list, feeds, pages, the Sources page's host select and its allow-list, feeds, pages, the Sources page's host select and its
@@ -14,21 +14,26 @@ since `desk.sources.registry` loads `plugins/*.py` by path and the tests
patch names on the `ats` module. The types are in `._contract`. patch names on the `ats` module. The types are in `._contract`.
A module is named after its source (`plugins/readers/workday.py`) and A module is named after its source (`plugins/readers/workday.py`) and
imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It imports `desk`, `._contract`, `._shared` and `._countries` only (besides the
holds: standard library, `httpx` and `pydantic`), never `plugins.ats` or another
reader; `tests/test_architecture.py` holds that. A board's module holds, as
Workday's does:
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
LABEL = "Workday" # what the Sources page's select shows LABEL = "Workday" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int; lower comes first in the select and the probes RANK = 100 # optional int; lower comes first in the select and the probes
KEEP_CASE = False # optional; True keeps a discovered slug's case KEEP_CASE = True # optional; True keeps a discovered slug's case
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000, READS = Reads(
text="detail", complete="stated", scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
filters=Filters(country="ids"), places="many", complete="stated", places="many",
fields=Fields(posted="listed"), filters=Filters(country="ids", remote="ids", query="one"),
robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)") fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots=("read: robots.txt allows /<Site>/ and disallows /refreshFacet/ and has no rule "
"on /wday/cxs/<tenant>/<site>/, the listing's POST .../jobs and each role's "
"JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"),
)
def slug_ok(slug: str) -> bool # strict, see below def slug_ok(slug: str) -> bool # strict, see below
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
@@ -49,24 +54,35 @@ and, optionally:
def fallback_url(slug: str) -> str # listing="whole": a smaller feed, def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
# read once when `feed_url` is over the cap # read once when `feed_url` is over the cap
def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt)
def probe_url(slug: str) -> str # discovery: 2xx only for a board def probe_url(slug: str) -> str # discovery: 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its **Search providers.** A module of `scope="search"` holds `NAME`, `LABEL`,
slug is the provider's own (a search reads no board), and it is not `allowed`, `READS`, the functions its listing needs, and:
discovered. A module without `READS` is read as a board with the older
signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by PARAMS = {"q": "query", "area": "place"} # saved param -> the filter dimension it fills
`undeclared()`; the conformance test marks it until it declares. def clean_params(params: dict[str, Any]) -> dict[str, Any]
`app/jobs/sources.search_params` keeps the params of `PARAMS` whose
dimension `READS.filters` declares and stores what `clean_params` returns;
`clean_params` raises ValueError, with a reason a person can read, for a
value the provider would not accept. The runner hands the saved params to
`page` in `Ask.params`. A search lists many companies, so its rows carry
`company`, the employer's name, which `plugins/search.py` stores. Its slug
is the provider's own (a search reads no board), and it has no patterns,
probes or careers page: it is never discovered.
**Declaration.** `READS` states what the reader does; the runner does it. **Declaration.** `READS` states what the reader does; the runner does it.
`problems()` checks it against the functions: `paged` needs `page` and `problems()` checks it against the functions: `paged` needs `page` and
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`, `rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs `text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key` `facets`, `scope="search"` needs `complete="never"` or `"stated"` and
names a config entry, `robots` is not empty, and each function takes the `PARAMS` naming filter dimensions, a `key` names a config entry, `robots`
arguments above. The traits: is not empty and carries a date, and each function takes the arguments
above. `tests/test_readers_contract.py` checks it against the reader's
fixtures: each declared field shows on some row, each field declared `none`
on none. The traits:
- `scope`: `board`, one company's whole list; `search`, many companies, - `scope`: `board`, one company's whole list; `search`, many companies,
never a whole list, so nothing a search stops listing is closed by it. never a whole list, so nothing a search stops listing is closed by it.
@@ -81,11 +97,13 @@ arguments above. The traits:
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`, - `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
`one` (one value a request), `many` (a list in one request), `ids` (host `one` (one value a request), `many` (a list in one request), `ids` (host
ids looked up with `facets`). ids looked up with `facets`).
- `places`: `one`, `many`, or `copies` when the host publishes one posting - `places`: `one`, `many`, `copies` when the host publishes one posting
per place with its own id. per place with its own id, or `none` when the host states no place.
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the - `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
posting date; `listed`, a date the host moves; `none`), and `remote`, posting date; `listed`, a date the host moves; `none`), and `remote`,
`country`, `seniority`, `employment` (`stated` or `none`). `country`, `seniority`, `employment` (`stated` or `none`). A field may
come from the listing or from `detail`: BambooHR, Gem and Rippling date a
role only on its own page.
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>` - `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
and its environment fallbacks. The runner resolves it and passes it in and its environment fallbacks. The runner resolves it and passes it in
`Ask.params["key"]`. `Ask.params["key"]`.
@@ -93,7 +111,7 @@ arguments above. The traits:
- `empty`: `suspect` when an unknown slug answers an empty list, so a read - `empty`: `suspect` when an unknown slug answers an empty list, so a read
far under the last count counts as failed. far under the last count counts as failed.
- `title`: `provisional` when the listing may lack the title and `detail` - `title`: `provisional` when the listing may lack the title and `detail`
fills it. fills it. Nothing reads it yet: `_runner.DETAIL_FIELDS` has no `title`.
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner **Ask.** `Ask(countries, places, remote, query, params)` is what the runner
wants from one series of requests. It sets only dimensions the reader wants from one series of requests. It sets only dimensions the reader
@@ -124,8 +142,7 @@ against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
paths are case-sensitive (Workday's site). A pattern without a group names paths are case-sensitive (Workday's site). A pattern without a group names
the host with no slug. A host that `discover.py` names in its own the host with no slug. A host that `discover.py` names in its own
`_UNREADABLE` table is taken from its module instead as soon as the module `_UNREADABLE` table loses its entry there when its module is added.
exists.
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host **Fetching.** `allowed(slug)` returns a predicate over a lowercased host
name; `_shared.only(host)` is the usual one. Every request goes through name; `_shared.only(host)` is the usual one. Every request goes through
@@ -165,9 +182,8 @@ see `desk.sources.strip_html`), `remote` (True, False, a word or None) and
`salary_text`; where the host states pay, `salary_min`, `salary_max` `salary_text`; where the host states pay, `salary_min`, `salary_max`
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`; (yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
where it states them, `country` (ISO 3166 alpha-2), `seniority` and where it states them, `country` (ISO 3166 alpha-2), `seniority` and
`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a `employment`; `teaser`; and, on a search row, `company`. The runner keeps
new field needs no change there. Undeclared modules do not state `places` every stated key in `raw`, so a new field needs no change there.
yet.
**Places.** A role has a list of places: `places` is `[Place(text=..., **Places.** A role has a list of places: `places` is `[Place(text=...,
remote=...)]`, one entry per place the host states, in the host's order, in remote=...)]`, one entry per place the host states, in the host's order, in
@@ -176,12 +192,14 @@ place and never joins places into one `text`. `remote` is True for a remote
place, False for an office, None when unsaid. A host that publishes one place, False for an office, None when unsaid. A host that publishes one
posting per place, each with its own id, declares `places="copies"`: each posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role copy is a row with its one place, and `dupes.py` folds them into one role
that keeps each copy's link. that keeps each copy's link. A host that states no place declares
`places="none"` and hands over `places == []`.
**Helpers.** `._shared` holds what more than one host needs. A module maps **Helpers.** `._shared` holds what more than one host needs, `._countries`
no word to the app's vocabulary: it hands on the host's own words under the country names hosts write. A module maps no word to the app's
`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them vocabulary: it hands on the host's own words under `seniority`,
through the host's table in `app/prep/hosts/<name>.py`. `employment`, `remote` and `country`, and `app.prep` maps them through the
host's table in `app/prep/hosts/<name>.py`.
- `place(text, remote=None)` makes a `Place`: remote when the text has the - `place(text, remote=None)` makes a `Place`: remote when the text has the
word `remote`, otherwise as `remote` says. `add_place(found, entry)` word `remote`, otherwise as `remote` says. `add_place(found, entry)`
@@ -197,38 +215,60 @@ is shown and embedded until `description` arrives. A reader that sees one
keeps it. keeps it.
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a **Detail.** `detail(get, slug, row)` reads one role's own page and returns a
dict of the fields it found. The shared code calls it for roles not yet read dict of the fields it found. The pull reads listings only; the `enrich`
(`ats:detailed:<target>`), at most `_runner.DETAIL_CAP` (200) a board a stage (`app/jobs/enrich.py`) calls `detail` after scoring, so the requests
pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that go where the ranking is: the open roles without text that an agent ranks
failed (`ats:detailfailed:<target>`). Only `_runner.DETAIL_FIELDS` are taken: among its best `enrich.top` at `enrich.min_score` or above, best first, at
most `enrich.per_board` from one board and `enrich.per_run` in all, one
batch per board through `ats.read_board_details` and
`_runner.read_details`, `_runner.PAGE_DELAY` apart, the roles never tried
before those whose read failed. Only `_runner.DETAIL_FIELDS` are taken:
`description`, `location`, `places`, `teaser`, `remote`, `country`, `description`, `location`, `places`, `teaser`, `remote`, `country`,
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the `employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys
listing's empty value; any other key is left out with a debug log line. whose columns survive a later pull that sends the listing's empty value;
any other key is left out with a debug log line, `seniority` among them.
`title`, `url`, `department` and `salary_text` cannot be filled from `title`, `url`, `department` and `salary_text` cannot be filled from
`detail`: the listing rewrites them every pull. A role not read in detail `detail`: the listing rewrites them every pull. A 404 or a 410 counts as read; any
on a pull is sent with an empty description, so the one read earlier stays. other failure is logged and retried on a later run; a read that gave no text
A 404 or a 410 counts as read; any other failure is logged and retried on a is retried after `enrich.retry_days`; a 429 ends the reads of that board for
later pull; a 429 ends the detail reads of that board for the pull. the run. A role read in detail is sent by later pulls without the listing's
A role read in detail on an earlier pull is sent without the place and remote, so what its own page said about them stays. Completeness
listing's place and remote, so what its own page said about them stays. is decided by the listing alone.
Completeness is decided by the listing alone.
**robots.txt.** Decided when the reader is built, not checked at run time. **robots.txt.** Decided when the reader is built, not checked at run time.
Whoever builds a reader reads the host's robots.txt for the paths the reader Whoever builds a reader reads the robots.txt of every host the reader
requests and reports what it disallows, and the owner decides whether the requests, for the paths it requests (the API's, not the career page a
host is read anyway. `robots` records that decision with its date, e.g. person opens), and the owner decides whether the host is read anyway.
`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is `robots` records what the file says of those paths, the date it was read
disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time; and the decision, e.g. `"read: robots.txt has no rule on /wday/cxs/
`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are (checked 2026-09-30)"` or `"read though /api/ is disallowed: owner,
read by nothing and a new reader defines neither. 2026-09-30"`; a record without a date fails the contract. Nothing reads
robots.txt at run time.
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when **Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
the slug is a board; `probe_ok(body)` narrows that when a host answers an the slug is a board; `probe_ok(body)` narrows that when a host answers an
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
name is probed as, `(guess,)` without it. A host without `probe_url` is never name is probed as, `(guess,)` without it. A host without `probe_url` is never
probed, only found by a link. probed, only found by a link.
"""
**Adding a reader.** A new host or provider touches:
- its module here;
- `app/prep/hosts/<name>.py` when it states seniority or employment in its
own words (the contract test requires the table; a `REMOTE` table when
its remote words are not the app's);
- its fixtures under `tests/fixtures/ats/` (`tests/fixtures/search/` for a
provider), named as `tests/test_readers_contract.py` describes, and
`tests/test_ats_<name>.py` and `tests/test_prep_hosts_<name>.py`;
- its entry in `discover._UNREADABLE`, deleted, where discovery named the
host before;
- a section in `plugins/README.md`, and a row in `docs/filter-survey.md`
when it filters at the source.
Nothing else: `HOSTS` and `PROVIDERS`, the Sources page's select,
discovery's patterns and probes, and the reach a board is read with are
derived from the module.
"""
from __future__ import annotations from __future__ import annotations
import importlib import importlib
+10
View File
@@ -32,6 +32,7 @@ already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels).
from __future__ import annotations from __future__ import annotations
import sys import sys
import textwrap
import types import types
from collections.abc import Callable from collections.abc import Callable
from pathlib import Path from pathlib import Path
@@ -535,6 +536,15 @@ def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
assert place_problems(rows, "many") == [] assert place_problems(rows, "many") == []
def test_the_contracts_example_is_workdays_declaration() -> None:
doc = readers.__doc__ or ""
start = doc.index(" READS = Reads(")
source = textwrap.dedent(doc[start:doc.index("\n )\n", start) + 6])
namespace: dict[str, Any] = {"Reads": Reads, "Filters": Filters, "Fields": Fields}
exec(source, namespace)
assert namespace["READS"] == readers.READERS["workday"].READS
def test_the_contract_types_are_what_the_package_exports() -> None: def test_the_contract_types_are_what_the_package_exports() -> None:
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None} assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}