Correct the contract docstring: detail is read by enrich, the example is Workday's, the touch points of a new reader
Adds the search providers' PARAMS, clean_params and company, and a test that the example READS is workday.py's. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
72252a1c04
commit
2cc83b1d84
2 files changed
+105
-55
No files matched your search
+95
-55
@@ -3,8 +3,8 @@
|
|||||||
A reader registers by existing: every module in this package whose name does
|
A reader registers by existing: every module in this package whose name does
|
||||||
not start with `_` is loaded when the package is imported, checked against
|
not start with `_` is loaded when the package is imported, checked against
|
||||||
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
|
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
|
||||||
`HOSTS` holds those of `scope="board"` (and every module without `READS`),
|
`HOSTS` holds those of `scope="board"`, `PROVIDERS` those of
|
||||||
`PROVIDERS` those of `scope="search"`. A module that breaks the contract
|
`scope="search"`. A module that breaks the contract, or declares no `READS`,
|
||||||
fails the import with an ImportError naming the module and every problem,
|
fails the import with an ImportError naming the module and every problem,
|
||||||
rather than a pull that fails board by board. `plugins/ats.py` builds its
|
rather than a pull that fails board by board. `plugins/ats.py` builds its
|
||||||
allow-list, feeds, pages, the Sources page's host select and its
|
allow-list, feeds, pages, the Sources page's host select and its
|
||||||
@@ -14,21 +14,26 @@ since `desk.sources.registry` loads `plugins/*.py` by path and the tests
|
|||||||
patch names on the `ats` module. The types are in `._contract`.
|
patch names on the `ats` module. The types are in `._contract`.
|
||||||
|
|
||||||
A module is named after its source (`plugins/readers/workday.py`) and
|
A module is named after its source (`plugins/readers/workday.py`) and
|
||||||
imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It
|
imports `desk`, `._contract`, `._shared` and `._countries` only (besides the
|
||||||
holds:
|
standard library, `httpx` and `pydantic`), never `plugins.ats` or another
|
||||||
|
reader; `tests/test_architecture.py` holds that. A board's module holds, as
|
||||||
|
Workday's does:
|
||||||
|
|
||||||
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
||||||
LABEL = "Workday" # what the Sources page's select shows
|
LABEL = "Workday" # what the Sources page's select shows
|
||||||
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||||
RANK = 100 # optional int; lower comes first in the select and the probes
|
RANK = 100 # optional int; lower comes first in the select and the probes
|
||||||
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
KEEP_CASE = True # optional; True keeps a discovered slug's case
|
||||||
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
|
|
||||||
|
|
||||||
READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000,
|
READS = Reads(
|
||||||
text="detail", complete="stated",
|
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
|
||||||
filters=Filters(country="ids"), places="many",
|
complete="stated", places="many",
|
||||||
fields=Fields(posted="listed"),
|
filters=Filters(country="ids", remote="ids", query="one"),
|
||||||
robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)")
|
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
||||||
|
robots=("read: robots.txt allows /<Site>/ and disallows /refreshFacet/ and has no rule "
|
||||||
|
"on /wday/cxs/<tenant>/<site>/, the listing's POST .../jobs and each role's "
|
||||||
|
"JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"),
|
||||||
|
)
|
||||||
|
|
||||||
def slug_ok(slug: str) -> bool # strict, see below
|
def slug_ok(slug: str) -> bool # strict, see below
|
||||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
||||||
@@ -49,24 +54,35 @@ and, optionally:
|
|||||||
|
|
||||||
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
|
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
|
||||||
# read once when `feed_url` is over the cap
|
# read once when `feed_url` is over the cap
|
||||||
def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt)
|
|
||||||
def probe_url(slug: str) -> str # discovery: 2xx only for a board
|
def probe_url(slug: str) -> str # discovery: 2xx only for a board
|
||||||
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
||||||
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
|
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
|
||||||
|
|
||||||
A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its
|
**Search providers.** A module of `scope="search"` holds `NAME`, `LABEL`,
|
||||||
slug is the provider's own (a search reads no board), and it is not
|
`allowed`, `READS`, the functions its listing needs, and:
|
||||||
discovered. A module without `READS` is read as a board with the older
|
|
||||||
signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by
|
PARAMS = {"q": "query", "area": "place"} # saved param -> the filter dimension it fills
|
||||||
`undeclared()`; the conformance test marks it until it declares.
|
def clean_params(params: dict[str, Any]) -> dict[str, Any]
|
||||||
|
|
||||||
|
`app/jobs/sources.search_params` keeps the params of `PARAMS` whose
|
||||||
|
dimension `READS.filters` declares and stores what `clean_params` returns;
|
||||||
|
`clean_params` raises ValueError, with a reason a person can read, for a
|
||||||
|
value the provider would not accept. The runner hands the saved params to
|
||||||
|
`page` in `Ask.params`. A search lists many companies, so its rows carry
|
||||||
|
`company`, the employer's name, which `plugins/search.py` stores. Its slug
|
||||||
|
is the provider's own (a search reads no board), and it has no patterns,
|
||||||
|
probes or careers page: it is never discovered.
|
||||||
|
|
||||||
**Declaration.** `READS` states what the reader does; the runner does it.
|
**Declaration.** `READS` states what the reader does; the runner does it.
|
||||||
`problems()` checks it against the functions: `paged` needs `page` and
|
`problems()` checks it against the functions: `paged` needs `page` and
|
||||||
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
|
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
|
||||||
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
|
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
|
||||||
`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key`
|
`facets`, `scope="search"` needs `complete="never"` or `"stated"` and
|
||||||
names a config entry, `robots` is not empty, and each function takes the
|
`PARAMS` naming filter dimensions, a `key` names a config entry, `robots`
|
||||||
arguments above. The traits:
|
is not empty and carries a date, and each function takes the arguments
|
||||||
|
above. `tests/test_readers_contract.py` checks it against the reader's
|
||||||
|
fixtures: each declared field shows on some row, each field declared `none`
|
||||||
|
on none. The traits:
|
||||||
|
|
||||||
- `scope`: `board`, one company's whole list; `search`, many companies,
|
- `scope`: `board`, one company's whole list; `search`, many companies,
|
||||||
never a whole list, so nothing a search stops listing is closed by it.
|
never a whole list, so nothing a search stops listing is closed by it.
|
||||||
@@ -81,11 +97,13 @@ arguments above. The traits:
|
|||||||
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
|
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
|
||||||
`one` (one value a request), `many` (a list in one request), `ids` (host
|
`one` (one value a request), `many` (a list in one request), `ids` (host
|
||||||
ids looked up with `facets`).
|
ids looked up with `facets`).
|
||||||
- `places`: `one`, `many`, or `copies` when the host publishes one posting
|
- `places`: `one`, `many`, `copies` when the host publishes one posting
|
||||||
per place with its own id.
|
per place with its own id, or `none` when the host states no place.
|
||||||
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
|
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
|
||||||
posting date; `listed`, a date the host moves; `none`), and `remote`,
|
posting date; `listed`, a date the host moves; `none`), and `remote`,
|
||||||
`country`, `seniority`, `employment` (`stated` or `none`).
|
`country`, `seniority`, `employment` (`stated` or `none`). A field may
|
||||||
|
come from the listing or from `detail`: BambooHR, Gem and Rippling date a
|
||||||
|
role only on its own page.
|
||||||
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
|
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
|
||||||
and its environment fallbacks. The runner resolves it and passes it in
|
and its environment fallbacks. The runner resolves it and passes it in
|
||||||
`Ask.params["key"]`.
|
`Ask.params["key"]`.
|
||||||
@@ -93,7 +111,7 @@ arguments above. The traits:
|
|||||||
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
|
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
|
||||||
far under the last count counts as failed.
|
far under the last count counts as failed.
|
||||||
- `title`: `provisional` when the listing may lack the title and `detail`
|
- `title`: `provisional` when the listing may lack the title and `detail`
|
||||||
fills it.
|
fills it. Nothing reads it yet: `_runner.DETAIL_FIELDS` has no `title`.
|
||||||
|
|
||||||
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
|
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
|
||||||
wants from one series of requests. It sets only dimensions the reader
|
wants from one series of requests. It sets only dimensions the reader
|
||||||
@@ -124,8 +142,7 @@ against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
|
|||||||
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
|
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
|
||||||
paths are case-sensitive (Workday's site). A pattern without a group names
|
paths are case-sensitive (Workday's site). A pattern without a group names
|
||||||
the host with no slug. A host that `discover.py` names in its own
|
the host with no slug. A host that `discover.py` names in its own
|
||||||
`_UNREADABLE` table is taken from its module instead as soon as the module
|
`_UNREADABLE` table loses its entry there when its module is added.
|
||||||
exists.
|
|
||||||
|
|
||||||
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
||||||
name; `_shared.only(host)` is the usual one. Every request goes through
|
name; `_shared.only(host)` is the usual one. Every request goes through
|
||||||
@@ -165,9 +182,8 @@ see `desk.sources.strip_html`), `remote` (True, False, a word or None) and
|
|||||||
`salary_text`; where the host states pay, `salary_min`, `salary_max`
|
`salary_text`; where the host states pay, `salary_min`, `salary_max`
|
||||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
||||||
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
||||||
`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a
|
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
|
||||||
new field needs no change there. Undeclared modules do not state `places`
|
every stated key in `raw`, so a new field needs no change there.
|
||||||
yet.
|
|
||||||
|
|
||||||
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
||||||
remote=...)]`, one entry per place the host states, in the host's order, in
|
remote=...)]`, one entry per place the host states, in the host's order, in
|
||||||
@@ -176,12 +192,14 @@ place and never joins places into one `text`. `remote` is True for a remote
|
|||||||
place, False for an office, None when unsaid. A host that publishes one
|
place, False for an office, None when unsaid. A host that publishes one
|
||||||
posting per place, each with its own id, declares `places="copies"`: each
|
posting per place, each with its own id, declares `places="copies"`: each
|
||||||
copy is a row with its one place, and `dupes.py` folds them into one role
|
copy is a row with its one place, and `dupes.py` folds them into one role
|
||||||
that keeps each copy's link.
|
that keeps each copy's link. A host that states no place declares
|
||||||
|
`places="none"` and hands over `places == []`.
|
||||||
|
|
||||||
**Helpers.** `._shared` holds what more than one host needs. A module maps
|
**Helpers.** `._shared` holds what more than one host needs, `._countries`
|
||||||
no word to the app's vocabulary: it hands on the host's own words under
|
the country names hosts write. A module maps no word to the app's
|
||||||
`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them
|
vocabulary: it hands on the host's own words under `seniority`,
|
||||||
through the host's table in `app/prep/hosts/<name>.py`.
|
`employment`, `remote` and `country`, and `app.prep` maps them through the
|
||||||
|
host's table in `app/prep/hosts/<name>.py`.
|
||||||
|
|
||||||
- `place(text, remote=None)` makes a `Place`: remote when the text has the
|
- `place(text, remote=None)` makes a `Place`: remote when the text has the
|
||||||
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
|
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
|
||||||
@@ -197,38 +215,60 @@ is shown and embedded until `description` arrives. A reader that sees one
|
|||||||
keeps it.
|
keeps it.
|
||||||
|
|
||||||
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
||||||
dict of the fields it found. The shared code calls it for roles not yet read
|
dict of the fields it found. The pull reads listings only; the `enrich`
|
||||||
(`ats:detailed:<target>`), at most `_runner.DETAIL_CAP` (200) a board a
|
stage (`app/jobs/enrich.py`) calls `detail` after scoring, so the requests
|
||||||
pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that
|
go where the ranking is: the open roles without text that an agent ranks
|
||||||
failed (`ats:detailfailed:<target>`). Only `_runner.DETAIL_FIELDS` are taken:
|
among its best `enrich.top` at `enrich.min_score` or above, best first, at
|
||||||
|
most `enrich.per_board` from one board and `enrich.per_run` in all, one
|
||||||
|
batch per board through `ats.read_board_details` and
|
||||||
|
`_runner.read_details`, `_runner.PAGE_DELAY` apart, the roles never tried
|
||||||
|
before those whose read failed. Only `_runner.DETAIL_FIELDS` are taken:
|
||||||
`description`, `location`, `places`, `teaser`, `remote`, `country`,
|
`description`, `location`, `places`, `teaser`, `remote`, `country`,
|
||||||
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the
|
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys
|
||||||
listing's empty value; any other key is left out with a debug log line.
|
whose columns survive a later pull that sends the listing's empty value;
|
||||||
|
any other key is left out with a debug log line, `seniority` among them.
|
||||||
`title`, `url`, `department` and `salary_text` cannot be filled from
|
`title`, `url`, `department` and `salary_text` cannot be filled from
|
||||||
`detail`: the listing rewrites them every pull. A role not read in detail
|
`detail`: the listing rewrites them every pull. A 404 or a 410 counts as read; any
|
||||||
on a pull is sent with an empty description, so the one read earlier stays.
|
other failure is logged and retried on a later run; a read that gave no text
|
||||||
A 404 or a 410 counts as read; any other failure is logged and retried on a
|
is retried after `enrich.retry_days`; a 429 ends the reads of that board for
|
||||||
later pull; a 429 ends the detail reads of that board for the pull.
|
the run. A role read in detail is sent by later pulls without the listing's
|
||||||
A role read in detail on an earlier pull is sent without the
|
place and remote, so what its own page said about them stays. Completeness
|
||||||
listing's place and remote, so what its own page said about them stays.
|
is decided by the listing alone.
|
||||||
Completeness is decided by the listing alone.
|
|
||||||
|
|
||||||
**robots.txt.** Decided when the reader is built, not checked at run time.
|
**robots.txt.** Decided when the reader is built, not checked at run time.
|
||||||
Whoever builds a reader reads the host's robots.txt for the paths the reader
|
Whoever builds a reader reads the robots.txt of every host the reader
|
||||||
requests and reports what it disallows, and the owner decides whether the
|
requests, for the paths it requests (the API's, not the career page a
|
||||||
host is read anyway. `robots` records that decision with its date, e.g.
|
person opens), and the owner decides whether the host is read anyway.
|
||||||
`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is
|
`robots` records what the file says of those paths, the date it was read
|
||||||
disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time;
|
and the decision, e.g. `"read: robots.txt has no rule on /wday/cxs/
|
||||||
`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are
|
(checked 2026-09-30)"` or `"read though /api/ is disallowed: owner,
|
||||||
read by nothing and a new reader defines neither.
|
2026-09-30"`; a record without a date fails the contract. Nothing reads
|
||||||
|
robots.txt at run time.
|
||||||
|
|
||||||
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
||||||
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
||||||
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
|
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
|
||||||
name is probed as, `(guess,)` without it. A host without `probe_url` is never
|
name is probed as, `(guess,)` without it. A host without `probe_url` is never
|
||||||
probed, only found by a link.
|
probed, only found by a link.
|
||||||
"""
|
|
||||||
|
|
||||||
|
**Adding a reader.** A new host or provider touches:
|
||||||
|
|
||||||
|
- its module here;
|
||||||
|
- `app/prep/hosts/<name>.py` when it states seniority or employment in its
|
||||||
|
own words (the contract test requires the table; a `REMOTE` table when
|
||||||
|
its remote words are not the app's);
|
||||||
|
- its fixtures under `tests/fixtures/ats/` (`tests/fixtures/search/` for a
|
||||||
|
provider), named as `tests/test_readers_contract.py` describes, and
|
||||||
|
`tests/test_ats_<name>.py` and `tests/test_prep_hosts_<name>.py`;
|
||||||
|
- its entry in `discover._UNREADABLE`, deleted, where discovery named the
|
||||||
|
host before;
|
||||||
|
- a section in `plugins/README.md`, and a row in `docs/filter-survey.md`
|
||||||
|
when it filters at the source.
|
||||||
|
|
||||||
|
Nothing else: `HOSTS` and `PROVIDERS`, the Sources page's select,
|
||||||
|
discovery's patterns and probes, and the reach a board is read with are
|
||||||
|
derived from the module.
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels).
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
import textwrap
|
||||||
import types
|
import types
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -535,6 +536,15 @@ def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
|
|||||||
assert place_problems(rows, "many") == []
|
assert place_problems(rows, "many") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_contracts_example_is_workdays_declaration() -> None:
|
||||||
|
doc = readers.__doc__ or ""
|
||||||
|
start = doc.index(" READS = Reads(")
|
||||||
|
source = textwrap.dedent(doc[start:doc.index("\n )\n", start) + 6])
|
||||||
|
namespace: dict[str, Any] = {"Reads": Reads, "Filters": Filters, "Fields": Fields}
|
||||||
|
exec(source, namespace)
|
||||||
|
assert namespace["READS"] == readers.READERS["workday"].READS
|
||||||
|
|
||||||
|
|
||||||
def test_the_contract_types_are_what_the_package_exports() -> None:
|
def test_the_contract_types_are_what_the_package_exports() -> None:
|
||||||
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
|
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
|
||||||
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
|
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
|
||||||
|
|||||||
Reference in new issue
Block a user