diff --git a/README.md b/README.md index fe39995..a2cc615 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # roledesk -A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor, Workday, SuccessFactors, Oracle, Phenom, Eightfold and a dozen smaller hosts, one module each in `plugins/atshosts/`) and runs saved searches on boards that span companies (Jobindex, Adzuna), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make. +A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor, Workday, SuccessFactors, Oracle, Phenom, Eightfold and a dozen smaller hosts, one module each in `plugins/readers/`) and runs saved searches on boards that span companies (Jobindex, Adzuna), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make. Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeline/`), password (`auth/`) and deploy kit (`deploy/`), plus the plugin contract and tools from news (`desk/`). FastAPI, Jinja2, SQLite, `uv`, no build step. diff --git a/app/jobs/discover.py b/app/jobs/discover.py index b52b8bb..c4bd671 100644 --- a/app/jobs/discover.py +++ b/app/jobs/discover.py @@ -5,7 +5,7 @@ The seed list and the LinkedIn flow both start from a company name and a website, not from a board. This module gets from the one to the other: it fetches the site's careers page, or the pages the home page links to as careers, and looks for a link into one of the hosts in `HOSTS`. A readable -host (one with a module in `plugins/atshosts/`, which the `ats` plugin +host (one with a module in `plugins/readers/`, which the `ats` plugin parses) becomes a `board` target the pull can read; any other host is recorded as `ats`, with its slug where the URL carries one, so the company is kept as one the app cannot read yet rather than dropped. The patterns and @@ -48,7 +48,7 @@ from urllib.parse import urljoin import httpx from desk.sources import USER_AGENT -from plugins import atshosts +from plugins import readers log = logging.getLogger(__name__) @@ -58,7 +58,7 @@ MAX_BYTES = 2 << 20 MAX_PAGES = 6 #: The hosts the `ats` plugin parses; a match on one of these is a `board`. -READABLE: tuple[str, ...] = tuple(atshosts.HOSTS) +READABLE: tuple[str, ...] = tuple(readers.HOSTS) #: host -> pattern over a URL, for the hosts discovery names but the app #: cannot read. Group 1, where there is one, is the slug the host names the @@ -104,14 +104,14 @@ def _hosts(modules: Mapping[str, ModuleType], #: host -> pattern over a URL, every host discovery can name. The slug is #: group 1, or, for a pattern with more groups, the groups that matched #: joined by "/" (Lever's `eu/`). -HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE) +HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(readers.HOSTS, _UNREADABLE) #: The hosts whose slugs keep the case the URL wrote them in, by their #: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its #: listing and its robots.txt paths are case-sensitive. Every other slug is #: lowercased, which is how a page that writes `BoschGroup` and one that #: writes `boschgroup` name one board. -_KEEP_CASE = frozenset(name for name, module in atshosts.HOSTS.items() +_KEEP_CASE = frozenset(name for name, module in readers.HOSTS.items() if getattr(module, "KEEP_CASE", False)) #: What a Teamtailor feed carries and a page that only looks like one does not: @@ -257,14 +257,14 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]: #: rate limit that a run of guesses trips, and a Workable site links its #: board plainly anyway. _PROBES: dict[str, Callable[[str], str]] = { - name: module.probe_url for name, module in atshosts.HOSTS.items() + name: module.probe_url for name, module in readers.HOSTS.items() if hasattr(module, "probe_url")} def _probe_slugs(host: str, guess: str) -> tuple[str, ...]: """The slugs a guessed name is probed as on `host`: the name itself, and for Lever the name on its EU instance too (the module's `probe_slugs`).""" - module = atshosts.HOSTS[host] + module = readers.HOSTS[host] return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,) @@ -281,7 +281,7 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool: return False if not response.is_success: return False - module = atshosts.HOSTS[host] + module = readers.HOSTS[host] return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True diff --git a/app/models.py b/app/models.py index 2af2065..2feec97 100644 --- a/app/models.py +++ b/app/models.py @@ -663,7 +663,7 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason) #: read as when a role was posted, checked against live boards on #: 2026-09-29: #: -#: - `greenhouse`: `plugins.atshosts.greenhouse.parse` already prefers +#: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers #: `first_published` over `updated_at`. A live board (adyen) showed a job #: with `first_published` five months before `updated_at`, proving the #: first stays fixed while the second moves on an edit — the resettable diff --git a/docs/decisions.md b/docs/decisions.md index e17cb16..6255d8f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -100,7 +100,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **A board that fails on every fetch for 21 days closes its roles.** A board whose fetch raised was only logged, so a company that moved ATS kept its roles open for good behind a 404. The `ats` plugin now records `ats:failing:` in `state` with the time of the first failure in a row and the latest reason, and a fetch that answers removes it. Once the first failure is `STALE_DAYS` old, the same window a search role gets, the settle closes the board's roles; the next answer reopens what it lists through `upsert_role`, as for a board switched back on. The record is plugin state rather than a column because the plugin only sees `self.state`. `/api/companies` returns it as `failing_since` and `failing_reason`, the Companies list shows "not answering since", and `app companies check` prints it beside its live result. -**A host is a module in `plugins/atshosts/`, and registers by existing.** `plugins/ats.py` held eight hosts in six parallel tables and `app/jobs/discover.py` three more, so two people adding a host edited the same lines. Each host is now one file that states its name, label, URL patterns, slug check, allowed host names and either a feed and a parser or a pager; `ats.py` and `discover.py` build their tables from the package, ordered by the module's `RANK` so the select and the probes kept their order. The package sits beside `ats.py` and is not `plugins/ats/`, since `desk.sources.registry` globs `plugins/*.py` and the tests patch names on the `ats` module. The contract is the docstring of `plugins/atshosts/__init__.py`, and a module that breaks it fails the import by name. +**A host is a module in `plugins/readers/`, and registers by existing.** `plugins/ats.py` held eight hosts in six parallel tables and `app/jobs/discover.py` three more, so two people adding a host edited the same lines. Each host is now one file that states its name, label, URL patterns, slug check, allowed host names and either a feed and a parser or a pager; `ats.py` and `discover.py` build their tables from the package, ordered by the module's `RANK` so the select and the probes kept their order. The package sits beside `ats.py` and is not `plugins/ats/`, since `desk.sources.registry` globs `plugins/*.py` and the tests patch names on the `ats` module. The contract is the docstring of `plugins/readers/__init__.py`, and a module that breaks it fails the import by name. **A host is handed the fetch.** A module imports `desk` and `._shared` only and receives `get`, which `ats.py` binds to its own `_get` at call time. The allow-list, the redirect rule, the byte cap and the deadline stay in one place, and a test that patches `ats._get` reaches every host. `_get` refuses an IP address, a local or numeric host name and any port but 443 before it asks, because a Teamtailor or SuccessFactors slug is a host name somebody typed. diff --git a/docs/plan-hosts-wave-2.md b/docs/plan-hosts-wave-2.md index 49f0c2c..073cc42 100644 --- a/docs/plan-hosts-wave-2.md +++ b/docs/plan-hosts-wave-2.md @@ -28,11 +28,11 @@ The survey's record of each company is in `docs/plan-hosts/survey-own.json` (cla ## Work packages -Each owns its modules in `plugins/atshosts/`, `tests/test_ats_.py` and `tests/fixtures/ats/*`, and nothing else. The common rules of wave 1 apply: the live request limits, robots.txt read first, no key from a script, a final report with the targets to attach. +Each owns its modules in `plugins/readers/`, `tests/test_ats_.py` and `tests/fixtures/ats/*`, and nothing else. The common rules of wave 1 apply: the live request limits, robots.txt read first, no key from a script, a final report with the targets to attach. | WP | Model | Builds | Needs | |---|---|---|---| -| 9 Provisional title | Opus | the row key, `upsert_role`'s rule and its tests; owns `plugins/ats.py`, `plugins/atshosts/__init__.py`, `app/storage/store.py`, `tests/test_atshosts.py`, `tests/test_storage.py` | the first decision | +| 9 Provisional title | Opus | the row key, `upsert_role`'s rule and its tests; owns `plugins/ats.py`, `plugins/readers/__init__.py`, `app/storage/store.py`, `tests/test_readers.py`, `tests/test_storage.py` | the first decision | | 10 Sitemap reader | Sonnet | `sitemap`: job URLs from a sitemap, the role from the page's JSON-LD or microdata | WP9 | | 11 Board filters | Opus | `params` on a board subscription, handed to `page`; owns `plugins/ats.py`, `app/jobs/pull.py`, `app/jobs/sources.py`, `app/web/api.py`, `app/static/companies.js` and their tests | the second decision | | 12 Own APIs | Sonnet | Ørsted, Amazon, Vinted, Dynatrace, RELEX, Siteimprove, Bending Spoons | | diff --git a/plugins/README.md b/plugins/README.md index 74df3d0..50951c6 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -1,8 +1,8 @@ # Where roles come from -This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a host module is the docstring of `plugins/atshosts/__init__.py`, not this file. +This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a reader (a host module or a search provider) is the docstring of `plugins/readers/__init__.py`, with its types and the declaration check in `plugins/readers/_contract.py` and the conformance test in `tests/test_readers_contract.py`, not this file. -The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/atshosts/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`. +The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/readers/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`. ## The three ways a source is read diff --git a/plugins/__init__.py b/plugins/__init__.py index 94ff7ff..af8d314 100644 --- a/plugins/__init__.py +++ b/plugins/__init__.py @@ -4,7 +4,7 @@ A plugin file imports `desk` only — see `tests/test_architecture.py` — and its own helpers in this directory. This `__init__.py` makes the directory importable as `plugins.`, for the rest of roledesk (`jobs/pull.py`, `cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the -package `plugins/atshosts/`. `desk.sources.registry` still finds `ats.py` by +package `plugins/readers/`. `desk.sources.registry` still finds `ats.py` by path, not by this package; `ats.py` puts the repo root on `sys.path` itself -when the app has not, so that `plugins.atshosts` imports either way. +when the app has not, so that `plugins.readers` imports either way. """ diff --git a/plugins/ats.py b/plugins/ats.py index dfe716e..7725a9c 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -1,5 +1,5 @@ """ATS boards: a company's own careers page on one of the hosts in -`plugins/atshosts/`, one module per host. +`plugins/readers/`, one module per host. Every one of these hosts serves a company's own list of roles as public, unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that @@ -49,7 +49,7 @@ from desk.sources import ( ) try: - from plugins import atshosts + from plugins import readers except ModuleNotFoundError: # `desk.sources.registry` loads this file by its path, under a name of its # own, and the app puts the repo root on `sys.path` before it does; a @@ -57,16 +57,16 @@ except ModuleNotFoundError: # does not, and the host modules are found as the package beside this # file. So the root goes on the path then, and only then. sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - from plugins import atshosts -from plugins.atshosts._shared import Get, Row + from plugins import readers +from plugins.readers._shared import Get, Row # The names the parsers had here before they moved to their host modules, # kept for the callers and the tests that still reach for them on `ats`. -from plugins.atshosts._shared import ( # noqa: F401 +from plugins.readers._shared import ( # noqa: F401 Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount, ) -from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401 -from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401 +from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401 +from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401 log = logging.getLogger(__name__) @@ -101,7 +101,7 @@ DEADLINE = 60.0 #: host -> the name the host gives itself: the hosts this plugin reads, in #: the order a select offers them. What the Sources page's host select is #: rendered from, so it cannot offer a host `is_known_target` refuses. -HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()} +HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()} def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: @@ -110,7 +110,7 @@ def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: company's own site. Anything else is refused, at the first hop and at every redirect: a feed is external content, and a board row must never be able to point this fetch at an internal address.""" - module = atshosts.HOSTS.get(host) + module = readers.HOSTS.get(host) return module.allowed(slug) if module is not None else lambda h: False @@ -234,7 +234,7 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get: #: host -> parser, for the hosts whose module parses a whole body. Every one #: takes the raw response body and the slug. _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { - name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")} + name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")} # --------------------------------------------------------- the classifier @@ -365,11 +365,11 @@ def classify(department: str, title: str) -> str: def is_known_target(target: str) -> bool: - """`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's + """`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's `slug_ok` accepts. What the target validator refuses: `app companies add` and the pull job both call this before touching the network.""" host, sep, slug = target.partition(":") - module = atshosts.HOSTS.get(host) + module = readers.HOSTS.get(host) return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug) @@ -377,7 +377,7 @@ def careers_url(target: str) -> str: """The company's own careers page for `host:slug`, or "" for an unknown host. What `app companies add` stores alongside the board.""" host, _, slug = target.partition(":") - module = atshosts.HOSTS.get(host) + module = readers.HOSTS.get(host) return module.careers_url(slug) if module is not None else "" @@ -404,7 +404,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: A 429 after the first page stops the read with what it has, incomplete; one on the first page raises like any other failure.""" host, _, slug = target.partition(":") - module = atshosts.HOSTS[host] + module = readers.HOSTS[host] get = _getter(client, host, slug) rows: dict[str, Row] = {} seen: set[str] = set() @@ -449,7 +449,7 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") host, _, slug = target.partition(":") - module = atshosts.HOSTS[host] + module = readers.HOSTS[host] if hasattr(module, "page"): return _paged(client, target) get = _getter(client, host, slug) @@ -566,7 +566,7 @@ def _check_robots(client: httpx.Client, target: str, kept, so the next pull asks again. A 429 counts as unreachable although it is a 4xx, since it says "not now" rather than "there is none".""" host, _, slug = target.partition(":") - module = atshosts.HOSTS.get(host) + module = readers.HOSTS.get(host) if module is None or not hasattr(module, "robots_page") or not is_known_target(target): return page = module.robots_page(slug) @@ -657,7 +657,7 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: after a complete listing, so one cut short does not make the next pull read its missing roles again.""" host, _, slug = target.partition(":") - module = atshosts.HOSTS[host] + module = readers.HOSTS[host] get = _getter(client, host, slug) done, failing = set(detailed), set(failed) - set(detailed) @@ -804,7 +804,7 @@ class ATSSource(Source): detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}") detail_sets: tuple[set[str], set[str]] | None = None - if hasattr(atshosts.HOSTS.get(host), "detail"): + if hasattr(readers.HOSTS.get(host), "detail"): detailed, failed = (self._id_set(key) for key in detail_keys) rows, detailed, failed = _read_details(client, target, rows, complete, detailed, failed) @@ -833,7 +833,7 @@ class ATSSource(Source): "function": classify(row["department"], row["title"]), "remote": row["remote"], # Every parser sets salary_text, empty or not (see - # "salary_text" in `atshosts/_shared.py`); only + # "salary_text" in `readers/_shared.py`); only # the hosts that state structured pay also set the # normalised figures, so the rest leave them at # row.get's default of no figure. diff --git a/plugins/atshosts/__init__.py b/plugins/atshosts/__init__.py deleted file mode 100644 index 1d694f3..0000000 --- a/plugins/atshosts/__init__.py +++ /dev/null @@ -1,217 +0,0 @@ -"""The hosts the `ats` plugin reads, one module each. - -A host registers by existing: every module in this package whose name does -not start with `_` is loaded when the package is imported, checked against -the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A -module that breaks the contract fails the import with an ImportError naming -the module and what it lacks, rather than a pull that fails board by board. -`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host -select and its description from `HOSTS`, and `app/jobs/discover.py` its -patterns and probes. The package sits beside `ats.py` rather than being -`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and -the tests patch names on the `ats` module. - -A module is named after its host (`plugins/atshosts/workday.py`) and imports -`desk` and `._shared` only, never `plugins.ats`. It holds: - - NAME = "workday" # the host of a target `workday:`; equals the module's name - LABEL = "Workday" # what the Sources page's select shows - PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE - RANK = 100 # optional int; lower comes first in the select and the probes - KEEP_CASE = False # optional; True keeps a discovered slug's case - ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case - - def slug_ok(slug: str) -> bool # strict, see below - def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach - def careers_url(slug: str) -> str # the page a person opens - -and one of two ways to read a board: - - def feed_url(slug: str) -> str # the whole board in one answer, - def parse(body: bytes, slug: str) -> list[Row] # read with `parse` - - def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call, - def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page - -and, optionally: - - def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read - # once when `feed_url` is over the cap - def robots_page(slug: str) -> str # the URL robots.txt must allow - def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page - def probe_url(slug: str) -> str # discovery: 2xx only for a board - def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough - def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as - -**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts -hold 10 to 80, which keeps the order they had before there were modules, so -a new host comes after them unless it says otherwise. The order decides the -select's default and which host wins when discovery's guessed slug exists on -two. - -**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`, -which lets a trailing newline through) and refuses anything that could add a -path, a query, a fragment, a port or a user to the URLs built from it. -`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok` -(one DNS label) cover the usual cases; a slug with several parts joins them -with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`). - -**Patterns.** The slug is group 1, or, when a pattern has more than one -group, the groups that matched joined by "/". Only group 1 is checked -against discovery's words that name no board (`api`, `jobs`, `embed`, ...). -The slug is lowercased unless `KEEP_CASE` is True: set it when the host's -paths are case-sensitive (Workday's site), since robots.txt paths are too. A -pattern without a group names the host with no slug. A host that -`discover.py` names in its own `_UNREADABLE` table is taken from its module -instead as soon as the module exists. - -**Fetching.** `allowed(slug)` returns a predicate over a lowercased host -name; `_shared.only(host)` is the usual one. Every request goes through -`get`, which is `ats._get` bound to the board: - - get(url, *, method="GET", payload=None, headers=None) -> bytes - -A POST sends `payload` as JSON (`{}` when None); `headers` are added to the -default User-Agent and `Accept: application/json, text/xml`, and replace -either where they name it. `get` is https only on port 443, never to an IP -address or a local name (`localhost`, `.local`, `.internal`, ...), follows at -most three redirects each checked against `allowed(slug)`, cuts the body at -`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns -the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read -`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection -failure, and another exception (`ValueError` among them) for a refused host, -an oversized body or a broken redirect. A module lets these through; the -shared code logs, records and decides. - -**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page -handed to `rows`, its raw results as dicts with an `id` (counted, by distinct -id, to decide completeness), the total the host states or None, and the -cursor of the next page or None after the last. The first call gets cursor -None. A board with a total is complete when as many distinct ids were read; -one without, when the read ended on an empty page. A 429 after the first page -ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY` -apart. A paged module may also have `parse(body, slug)` for one page's body. - -**Rows.** A `Row` is a dict with `source_id` (the host's own id for the -role, unique on the board), `title`, `location`, `url` (http or https, -checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see -`_shared.parse_dt`), `department`, `description` (plain text, see -`desk.sources.strip_html`), `remote` (True, False, a word or None) and -`salary_text`; and, where the host states pay, `salary_min`, `salary_max` -(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`. - -**Detail.** `detail(get, slug, row)` reads one role's own page and returns a -dict of the fields it found. The shared code calls it for roles not yet read -(`ats:detailed:`), at most `ats.DETAIL_CAP` (200) a board a pull, -`ats.PAGE_DELAY` apart, the roles never tried before those that failed -(`ats:detailfailed:`). Only `ats.DETAIL_FIELDS` are taken: -`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` -and `posted`, the keys whose columns survive a later pull that sends the -listing's empty value; any other key is left out with a debug log line. -`title`, `url`, `department` and `salary_text` cannot be filled from -`detail`: the listing rewrites them every pull. A role not read in detail -on a pull is sent with an empty description, so the one read earlier stays. -A 404 or a 410 counts as read; any other failure is logged and retried on a -later pull; a 429 ends the detail reads of that board for the pull. -A role read in detail on an earlier pull is sent without the -listing's place and remote, so what its own page said about them stays. -Completeness is decided by the listing alone. - -**robots.txt.** With `robots_page(slug)`, the shared code reads -`https:///robots.txt` through `get` before the listing, at most -once a day per board (`ats:robots:`), following a redirect to the -host's `www.` twin or back and no other. The board fails with "robots.txt -disallows it" when the rules for `*` or for the app's own token disallow the -page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a -final `$` in paths, case-sensitive paths, or without case for a host with -`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as -Workday does). A 4xx other than 429 means no -rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails -the board with "robots.txt could not be read" and is asked again next pull. -`robots_page` must be on a host `allowed(slug)` admits. - -**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when -the slug is a board; `probe_ok(body)` narrows that when a host answers an -empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed -name is probed as, `(guess,)` without it. A host without `probe_url` is never -probed, only found by a link. -""" - -from __future__ import annotations - -import importlib -import pkgutil -import re -from collections.abc import Iterable -from types import ModuleType - -#: The rank of a module that states none: after the eight first hosts, whose -#: ranks (10 to 80) keep the order they had before there were modules. -DEFAULT_RANK = 100 - -#: What every host module defines. -REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url") - -#: The functions a module may define; each must be callable if present. -FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows", - "fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs") - - -def problems(module: ModuleType, name: str | None = None) -> list[str]: - """What `module` lacks or gets wrong against the contract, one phrase - each; empty for a module that keeps it. `name` is the module's own name - in the package, which `NAME` must equal.""" - out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)] - if name is not None and getattr(module, "NAME", name) != name: - out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}") - if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL): - out.append("LABEL is not a non-empty string") - if not isinstance(getattr(module, "KEEP_CASE", False), bool): - out.append("KEEP_CASE is not a bool") - if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool): - out.append("ROBOTS_ANY_CASE is not a bool") - rank = getattr(module, "RANK", DEFAULT_RANK) - if not isinstance(rank, int) or isinstance(rank, bool): - out.append("RANK is not an int") - patterns = getattr(module, "PATTERNS", ()) - if isinstance(patterns, str) or not isinstance(patterns, Iterable): - out.append("PATTERNS is not a tuple of strings") - else: - for pattern in patterns: - try: - re.compile(pattern) - except (TypeError, re.error) as exc: - out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})") - out += [f"{attr} is not callable" for attr in FUNCTIONS - if hasattr(module, attr) and not callable(getattr(module, attr))] - if hasattr(module, "page"): - if not hasattr(module, "rows"): - out.append("page without rows") - elif not (hasattr(module, "feed_url") and hasattr(module, "parse")): - out.append("neither feed_url and parse nor page and rows") - if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"): - out.append("probe_ok without probe_url") - if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"): - out.append("probe_slugs without probe_url") - return out - - -def load(path: Iterable[str], package: str) -> dict[str, ModuleType]: - """`NAME -> module` for every module of `package` (found on `path`) whose - name does not start with `_`, in order of `(RANK, NAME)`. Raises - ImportError naming the first module that breaks the contract.""" - modules: list[ModuleType] = [] - for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name): - if info.name.startswith("_"): - continue - module = importlib.import_module(f"{package}.{info.name}") - if found := problems(module, info.name): - raise ImportError(f"{package}.{info.name} breaks the host contract: " - + "; ".join(found)) - modules.append(module) - modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME)) - return {module.NAME: module for module in modules} - - -#: host -> its module, in order of `(RANK, NAME)`. -HOSTS: dict[str, ModuleType] = load(__path__, __name__) diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py new file mode 100644 index 0000000..29021ff --- /dev/null +++ b/plugins/readers/__init__.py @@ -0,0 +1,329 @@ +"""The readers: one module per source of roles, a host's boards or a search. + +A reader registers by existing: every module in this package whose name does +not start with `_` is loaded when the package is imported, checked against +the contract below, and listed in `READERS` in order of `(RANK, NAME)`; +`HOSTS` holds those of `scope="board"` (and every module without `READS`), +`PROVIDERS` those of `scope="search"`. A module that breaks the contract +fails the import with an ImportError naming the module and every problem, +rather than a pull that fails board by board. `plugins/ats.py` builds its +allow-list, feeds, pages, the Sources page's host select and its +description from `HOSTS`, and `app/jobs/discover.py` its patterns and +probes. The package sits beside `ats.py` rather than being `plugins/ats/`, +since `desk.sources.registry` loads `plugins/*.py` by path and the tests +patch names on the `ats` module. The types are in `._contract`. + +A module is named after its source (`plugins/readers/workday.py`) and +imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It +holds: + + NAME = "workday" # the host of a target `workday:`; equals the module's name + LABEL = "Workday" # what the Sources page's select shows + PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE + RANK = 100 # optional int; lower comes first in the select and the probes + KEEP_CASE = False # optional; True keeps a discovered slug's case + ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case + + READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000, + text="detail", complete="stated", + filters=Filters(country="ids"), places="many", + fields=Fields(posted="listed"), + robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)") + + def slug_ok(slug: str) -> bool # strict, see below + def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach + def careers_url(slug: str) -> str # the page a person opens + +and the functions its `READS` names: + + def feed_url(slug: str, ask: Ask) -> str # listing="whole": one answer, + def parse(body: bytes, slug: str) -> list[Row] # read with `parse` + + def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged", + def rows(decoded: Any, slug: str) -> list[Row] # a page a call + + def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail" + def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter + +and, optionally: + + def fallback_url(slug: str) -> str # listing="whole": a smaller feed, + # read once when `feed_url` is over the cap + def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt) + def probe_url(slug: str) -> str # discovery: 2xx only for a board + def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough + def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as + +A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its +slug is the provider's own (a search reads no board), and it is not +discovered. A module without `READS` is read as a board with the older +signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by +`undeclared()`; the conformance test marks it until it declares. + +**Declaration.** `READS` states what the reader does; the runner does it. +`problems()` checks it against the functions: `paged` needs `page` and +`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`, +`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs +`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key` +names a config entry, `robots` is not empty, and each function takes the +arguments above. The traits: + +- `scope`: `board`, one company's whole list; `search`, many companies, + never a whole list, so nothing a search stops listing is closed by it. +- `listing`: `whole`, one request; `paged`, with `page_size` and `ceiling`, + the offset past which the host answers nothing whatever is asked. A board + above its ceiling is incomplete unless a filter brings it under. +- `text`: `listing`, the list has the full text; `excerpt`, a teaser and no + more; `detail`, one request a role; `none`. `teaser=True` with `detail` + says the list carries a teaser as well. +- `complete`: `stated`, the host gives a total; `end`, an empty page ends + the list; `whole`, the one answer is all; `never`. +- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`, + `one` (one value a request), `many` (a list in one request), `ids` (host + ids looked up with `facets`). +- `places`: `one`, `many`, or `copies` when the host publishes one posting + per place with its own id. +- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the + posting date; `listed`, a date the host moves; `none`), and `remote`, + `country`, `seniority`, `employment` (`stated` or `none`). +- `key`: `Key(name, fields, env)`, the config entry `sources.search.` + and its environment fallbacks. The runner resolves it and passes it in + `Ask.params["key"]`. +- `robots`: the robots.txt decision, see below. Required. +- `empty`: `suspect` when an unknown slug answers an empty list, so a read + far under the last count counts as failed. +- `title`: `provisional` when the listing may lack the title and `detail` + fills it. + +**Ask.** `Ask(countries, places, remote, query, params)` is what the runner +wants from one series of requests. It sets only dimensions the reader +declared. With `one` it runs a series per value and unions the rows by +`source_id`. With `ids`, `countries` and `places` hold host ids, which the +runner maps from `facets(get, slug)`: `{"country": {"DE": ""}, +"place": {...}}`, called at most once a day per board and kept in state. +`params` holds a search's saved params. `Ask()` is an unfiltered read. A +reader turns `Ask` into its host's query in `feed_url` or `page` and never +filters rows itself. + +**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts +hold 10 to 80, which keeps the order they had before there were modules, so +a new host comes after them unless it says otherwise. The order decides the +select's default and which host wins when discovery's guessed slug exists on +two. + +**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`, +which lets a trailing newline through) and refuses anything that could add a +path, a query, a fragment, a port or a user to the URLs built from it. +`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok` +(one DNS label) cover the usual cases; a slug with several parts joins them +with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`). + +**Patterns.** The slug is group 1, or, when a pattern has more than one +group, the groups that matched joined by "/". Only group 1 is checked +against discovery's words that name no board (`api`, `jobs`, `embed`, ...). +The slug is lowercased unless `KEEP_CASE` is True: set it when the host's +paths are case-sensitive (Workday's site). A pattern without a group names +the host with no slug. A host that `discover.py` names in its own +`_UNREADABLE` table is taken from its module instead as soon as the module +exists. + +**Fetching.** `allowed(slug)` returns a predicate over a lowercased host +name; `_shared.only(host)` is the usual one. Every request goes through +`get`, which is `ats._get` bound to the board: + + get(url, *, method="GET", payload=None, headers=None) -> bytes + +A POST sends `payload` as JSON (`{}` when None); `headers` are added to the +default User-Agent and `Accept: application/json, text/xml`, and replace +either where they name it. `get` is https only on port 443, never to an IP +address or a local name (`localhost`, `.local`, `.internal`, ...), follows at +most three redirects each checked against `allowed(slug)`, cuts the body at +`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns +the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read +`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection +failure, and another exception (`ValueError` among them) for a refused host, +an oversized body or a broken redirect. A module lets these through; the +shared code logs, records and decides. + +**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page +handed to `rows`, its raw results as dicts with an `id` (counted, by distinct +id, to decide completeness), the total the host states or None, and the +cursor of the next page or None after the last. The first call gets cursor +None. A board with a total is complete when as many distinct ids were read; +one without, when the read ended on an empty page. A 429 after the first page +ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY` +apart. A paged module may also have `parse(body, slug)` for one page's body. + +**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's +own id for the role, unique on the board), `title`, `url` (http or https, +checked with `_shared.is_web_url`) and `places`; `location` (the host's +one-line place, kept while storage reads it), `posted` (an aware datetime +or None, see `_shared.parse_dt`), `department`, `description` (plain text, +see `desk.sources.strip_html`), `remote` (True, False, a word or None) and +`salary_text`; where the host states pay, `salary_min`, `salary_max` +(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`; +where it states them, `country` (ISO 3166 alpha-2), `seniority` and +`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a +new field needs no change there. Undeclared modules do not state `places` +yet. + +**Places.** A role has a list of places: `places` is `[Place(text=..., +remote=...)]`, one entry per place the host states, in the host's order, in +the host's words. A reader never multiplies a row per place, never drops a +place and never joins places into one `text`. `remote` is True for a remote +place, False for an office, None when unsaid. A host that publishes one +posting per place, each with its own id, declares `places="copies"`: each +copy is a row with its one place, and `dupes.py` folds them into one role +that keeps each copy's link. + +**Teaser.** A short text the list carries (an excerpt, a summary) is handed +over as `teaser`, plain text, whether or not the full text comes later. It +is shown and embedded until `description` arrives. A reader that sees one +keeps it. + +**Detail.** `detail(get, slug, row)` reads one role's own page and returns a +dict of the fields it found. The shared code calls it for roles not yet read +(`ats:detailed:`), at most `ats.DETAIL_CAP` (200) a board a pull, +`ats.PAGE_DELAY` apart, the roles never tried before those that failed +(`ats:detailfailed:`). Only `ats.DETAIL_FIELDS` are taken: +`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` +and `posted`, the keys whose columns survive a later pull that sends the +listing's empty value; any other key is left out with a debug log line. +`title`, `url`, `department` and `salary_text` cannot be filled from +`detail`: the listing rewrites them every pull. A role not read in detail +on a pull is sent with an empty description, so the one read earlier stays. +A 404 or a 410 counts as read; any other failure is logged and retried on a +later pull; a 429 ends the detail reads of that board for the pull. +A role read in detail on an earlier pull is sent without the +listing's place and remote, so what its own page said about them stays. +Completeness is decided by the listing alone. + +**robots.txt.** Decided when the reader is built, not checked at run time. +Whoever builds a reader reads the host's robots.txt for the paths the reader +requests and reports what it disallows, and the owner decides whether the +host is read anyway. `robots` records that decision with its date, e.g. +`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is +disallowed: owner, 2026-09-30"`. The runtime check in `ats.py`, which +`robots_page` and `ROBOTS_ANY_CASE` feed, predates this rule and goes with +the runner; a new reader defines neither. Until then it reads +`https:///robots.txt` at most once a day per board and fails the board +when the page is disallowed for `*` or the app's token. + +**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when +the slug is a board; `probe_ok(body)` narrows that when a host answers an +empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed +name is probed as, `(guess,)` without it. A host without `probe_url` is never +probed, only found by a link. +""" + +from __future__ import annotations + +import importlib +import pkgutil +import re +from collections.abc import Iterable +from types import ModuleType + +from . import _contract +from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row + +__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key", + "Place", "Reads", "Row", "load", "problems", "scope", "undeclared"] + +#: The rank of a module that states none: after the eight first hosts, whose +#: ranks (10 to 80) keep the order they had before there were modules. +DEFAULT_RANK = 100 + +#: What every board module defines. +REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url") + +#: What a module of `scope="search"` defines. +REQUIRED_SEARCH = ("NAME", "LABEL", "allowed") + +#: The functions a module may define; each must be callable if present. +FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows", + "fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok", + "probe_slugs") + + +def scope(module: ModuleType) -> str: + """`board` or `search`; `board` for a module without `READS`.""" + reads = getattr(module, "READS", None) + return reads.scope if isinstance(reads, Reads) else "board" + + +def problems(module: ModuleType, name: str | None = None) -> list[str]: + """What `module` lacks or gets wrong against the contract, one phrase + each; empty for a module that keeps it. `name` is the module's own name + in the package, which `NAME` must equal. `READS` is checked when the + module declares one (`_contract.problems`).""" + required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED + out = [f"no {attr}" for attr in required if not hasattr(module, attr)] + if name is not None and getattr(module, "NAME", name) != name: + out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}") + if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL): + out.append("LABEL is not a non-empty string") + if not isinstance(getattr(module, "KEEP_CASE", False), bool): + out.append("KEEP_CASE is not a bool") + if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool): + out.append("ROBOTS_ANY_CASE is not a bool") + rank = getattr(module, "RANK", DEFAULT_RANK) + if not isinstance(rank, int) or isinstance(rank, bool): + out.append("RANK is not an int") + patterns = getattr(module, "PATTERNS", ()) + if isinstance(patterns, str) or not isinstance(patterns, Iterable): + out.append("PATTERNS is not a tuple of strings") + else: + for pattern in patterns: + try: + re.compile(pattern) + except (TypeError, re.error) as exc: + out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})") + out += [f"{attr} is not callable" for attr in FUNCTIONS + if hasattr(module, attr) and not callable(getattr(module, attr))] + if hasattr(module, "page"): + if not hasattr(module, "rows"): + out.append("page without rows") + elif not (hasattr(module, "feed_url") and hasattr(module, "parse")): + out.append("neither feed_url and parse nor page and rows") + if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"): + out.append("probe_ok without probe_url") + if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"): + out.append("probe_slugs without probe_url") + return out + _contract.problems(module) + + +def load(path: Iterable[str], package: str) -> dict[str, ModuleType]: + """`NAME -> module` for every module of `package` (found on `path`) whose + name does not start with `_`, in order of `(RANK, NAME)`. Raises + ImportError naming the first module that breaks the contract.""" + modules: list[ModuleType] = [] + for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name): + if info.name.startswith("_"): + continue + module = importlib.import_module(f"{package}.{info.name}") + if found := problems(module, info.name): + raise ImportError(f"{package}.{info.name} breaks the reader contract: " + + "; ".join(found)) + modules.append(module) + modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME)) + return {module.NAME: module for module in modules} + + +def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]: + """The names of the modules (of `READERS` by default) without `READS`, + read with the older signatures until they declare.""" + return [name for name, module in (READERS if modules is None else modules).items() + if not hasattr(module, "READS")] + + +#: name -> module for every reader, in order of `(RANK, NAME)`. +READERS: dict[str, ModuleType] = load(__path__, __name__) + +#: host -> its module: the readers of `scope="board"` and the undeclared. +HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items() + if scope(module) == "board"} + +#: provider -> its module: the readers of `scope="search"`. +PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items() + if scope(module) == "search"} diff --git a/plugins/readers/_contract.py b/plugins/readers/_contract.py new file mode 100644 index 0000000..250d494 --- /dev/null +++ b/plugins/readers/_contract.py @@ -0,0 +1,291 @@ +"""The types of the reader contract, and the check of a module's declaration +against its functions. The prose is the package's docstring; this file is +what it names. + +A reader declares `READS = Reads(...)`: what it reads (a board or a search), +how the listing arrives, where the text is, how completeness is known, what +it can filter at the source, how it states places and which fields it +states. The runner reads the declaration and does the rest. Values are +checked by pydantic where the module is declared, so a misspelt trait fails +the import; what depends on the module's functions, or on two traits at +once, is `problems()`, which the package's loader runs and turns into an +ImportError naming every problem. + +Signatures of a module that declares `READS`, with `Get` and `Page` from +`._shared`: + + def feed_url(slug: str, ask: Ask) -> str # listing="whole" + def parse(body: bytes, slug: str) -> list[Row] # listing="whole" + def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged" + def rows(decoded: Any, slug: str) -> list[Row] # listing="paged" + def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail" + def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter + +A module without `READS` keeps the signatures without `ask` until it +declares; `undeclared()` in the package lists those modules. +""" + +from __future__ import annotations + +import inspect +import re +from collections.abc import Callable +from types import ModuleType +from typing import Any, Literal, Required, TypedDict + +from pydantic import ConfigDict + +from desk.models import Model + +#: A filter dimension's reach at the source: `none`, not filtered; `one`, one +#: value a request (the runner makes one series per value and unions them by +#: id); `many`, a list in one request; `ids`, values the host names by its own +#: ids, which `facets` looks up. +Filter = Literal["none", "one", "many", "ids"] + +#: Whether the reader states a field: `stated` from the host, `none` never. +Stated = Literal["stated", "none"] + +#: A config entry name: `sources.search.` in `config.yaml`. +_CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*") + +#: An environment variable name. +_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*") + + +class _Frozen(Model): + """A declaration: immutable, extra keys refused.""" + + model_config = ConfigDict(from_attributes=True, extra="forbid", frozen=True) + + +class Filters(_Frozen): + """What the host filters at the source, per dimension. `country` takes + ISO 3166 alpha-2 codes, `place` place names, `remote` a bool, `query` + words. The runner never hands a reader a value on a dimension it + declared `none`.""" + + country: Filter = "none" + place: Filter = "none" + remote: Filter = "none" + query: Filter = "none" + + def declared(self) -> dict[str, Filter]: + """The dimensions filtered at the source, `name -> reach`.""" + return {name: reach for name, reach in self.model_dump().items() if reach != "none"} + + +class Fields(_Frozen): + """Which fields the rows state, and how far they can be trusted. + + `pay`: `figures` (`salary_min`, `salary_max`, `currency`), `line` (only + `salary_text`), `none`. `posted`: `trusted` (the host's date is the + posting date), `listed` (a date, but one the host moves, e.g. on an + edit), `none`. The rest: `stated` when the host states it, `none` when + the app has to infer it. + """ + + pay: Literal["figures", "line", "none"] = "none" + posted: Literal["trusted", "listed", "none"] = "none" + remote: Stated = "none" + country: Stated = "none" + seniority: Stated = "none" + employment: Stated = "none" + + +class Key(_Frozen): + """An API key the reader needs: the config entry `sources.search.` + with `fields`, each falling back to the environment variable at the same + position in `env`. The runner resolves it and hands the reader the + values in `Ask.params["key"]`; a reader never reads a key itself.""" + + name: str + fields: tuple[str, ...] + env: tuple[str, ...] = () + + +class Reads(_Frozen): + """What a reader reads and how. One per module, as `READS`. + + `scope`: `board` (one company's whole list) or `search` (many companies, + never a whole list). `listing`: `whole` (one answer) or `paged`. + `page_size` and `ceiling` (paged only): roles a page, and the offset + where the host stops answering whatever is asked (Workday 2,000). + `text`: `listing` (full text in the list), `excerpt` (a teaser in the + list, nothing more), `detail` (one request a role), `none`. `teaser`: + with `text="detail"`, the list also carries a teaser, handed over as + `Row.teaser`. `complete`: `stated` (a total), `end` (an empty page), + `whole` (the one answer), `never` (a search). `places`: `one` (the host + states one), `many` (a list per role), `copies` (the host lists one + posting per place, each with its own id). `robots`: the robots.txt + decision, with the date, taken when the reader was built. `empty`: + `suspect` when an unknown slug answers an empty list rather than an + error. `title`: `provisional` when the listing may not state the title + (a sitemap), which detail then fills. + """ + + scope: Literal["board", "search"] + listing: Literal["whole", "paged"] + page_size: int | None = None + ceiling: int | None = None + text: Literal["listing", "excerpt", "detail", "none"] + teaser: bool = False + complete: Literal["stated", "end", "whole", "never"] + filters: Filters = Filters() + places: Literal["one", "many", "copies"] + fields: Fields = Fields() + robots: str + empty: Literal["trusted", "suspect"] = "trusted" + key: Key | None = None + title: Literal["final", "provisional"] = "final" + + +class Ask(_Frozen): + """What the runner wants from one series of requests. Only the + dimensions the reader declared are set; for a `one` filter each series + carries one value. With `ids`, `countries` and `places` hold the host's + ids from `facets`, not codes or names. `params` are a search's saved + params (and a resolved key under `"key"`). An empty `Ask()` is an + unfiltered read.""" + + countries: tuple[str, ...] = () + places: tuple[str, ...] = () + remote: bool | None = None + query: str = "" + params: dict[str, Any] = {} + + +class Place(TypedDict): + """One place of a role, as the host states it: `text` the host's own + words ("Berlin, Germany"), `remote` True for a remote place ("Remote + (DE)"), False for an office, None when the host does not say. Resolving + `text` to a city, region or country is the app's job, not the reader's.""" + + text: str + remote: bool | None + + +class Row(TypedDict, total=False): + """One role as a reader hands it over. + + `source_id` is the host's id, unique within the board or the search; + `url` a web URL (`_shared.is_web_url`); `places` every place the host + states, never multiplied into rows, never cut to one; `teaser` the + short text the list carries. `location` is the host's own one-line + place, kept while storage reads it. `posted` is an aware datetime or + None; `description` plain text; `remote` True, False, a word or None; + pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and + `employment` only where the host states them, as `Fields` declares. Any + other stated key is kept in `raw` by the runner. + """ + + source_id: Required[str] + title: Required[str] + url: Required[str] + places: Required[list[Place]] + teaser: str + location: str + posted: Any + department: str + description: str + remote: bool | str | None + salary_text: str + salary_min: int | None + salary_max: int | None + currency: str | None + country: str | None + seniority: str | None + employment: str | None + + +#: What a declared module's functions take, positionally. +SIGNATURES: dict[str, tuple[str, ...]] = { + "feed_url": ("slug", "ask"), + "parse": ("body", "slug"), + "page": ("get", "slug", "cursor", "ask"), + "rows": ("decoded", "slug"), + "detail": ("get", "slug", "row"), + "facets": ("get", "slug"), +} + + +def _takes(function: Callable[..., Any], count: int) -> bool: + """Whether `function` accepts `count` positional arguments.""" + try: + signature = inspect.signature(function) + except (TypeError, ValueError): + return True + try: + signature.bind(*([None] * count)) + except TypeError: + return False + return True + + +def problems(module: ModuleType) -> list[str]: + """What `module`'s `READS` gets wrong, on its own or against the + module's functions; empty for a module that declares none.""" + if not hasattr(module, "READS"): + return [] + reads = module.READS + if not isinstance(reads, Reads): + return ["READS is not a Reads"] + has = lambda name: callable(getattr(module, name, None)) # noqa: E731 + out: list[str] = [] + + if reads.listing == "paged" and not (has("page") and has("rows")): + out.append('listing="paged" without page and rows') + if reads.listing == "whole" and not (has("feed_url") and has("parse")): + out.append('listing="whole" without feed_url and parse') + if reads.listing == "whole" and has("page"): + out.append('listing="whole" with page') + if reads.listing == "whole" and (reads.page_size is not None or reads.ceiling is not None): + out.append('page_size and ceiling are for listing="paged"') + if reads.page_size is not None and reads.page_size < 1: + out.append("page_size is not positive") + if reads.ceiling is not None and reads.ceiling < (reads.page_size or 1): + out.append("ceiling is below page_size") + if has("fallback_url") and reads.listing != "whole": + out.append('fallback_url without listing="whole"') + + if reads.text == "detail" and not has("detail"): + out.append('text="detail" without detail') + if has("detail") and reads.text != "detail": + out.append(f'detail with text="{reads.text}"') + if reads.title == "provisional" and not has("detail"): + out.append('title="provisional" without detail to fill it') + if reads.teaser and reads.text != "detail": + out.append('teaser is for text="detail"; the list of "excerpt" is the teaser') + + if (reads.complete == "whole") != (reads.listing == "whole"): + out.append('complete="whole" goes with listing="whole" and only with it') + if reads.scope == "search" and reads.complete not in ("never", "stated"): + out.append('scope="search" needs complete="never" or "stated"') + if reads.scope == "board" and reads.complete == "never": + out.append('scope="board" cannot be complete="never"') + + ids = [name for name, reach in reads.filters.declared().items() if reach == "ids"] + if ids and not has("facets"): + out.append(f"an ids filter ({', '.join(ids)}) without facets") + if has("facets") and not ids: + out.append("facets without an ids filter") + if reads.filters.query == "ids": + out.append('query cannot be "ids"') + + if key := reads.key: + if not _CONFIG_NAME.fullmatch(key.name): + out.append(f"key names no config entry ({key.name!r})") + if not key.fields: + out.append("key has no fields") + if key.env and len(key.env) != len(key.fields): + out.append("key.env does not match key.fields") + if any(not _ENV_NAME.fullmatch(name) for name in key.env): + out.append("key.env has a name that is not an environment variable") + + if not reads.robots.strip(): + out.append("robots is empty: record the robots.txt decision and its date") + + for name, params in SIGNATURES.items(): + if has(name) and not _takes(getattr(module, name), len(params)): + out.append(f"{name} does not take ({', '.join(params)})") + return out diff --git a/plugins/atshosts/_shared.py b/plugins/readers/_shared.py similarity index 97% rename from plugins/atshosts/_shared.py rename to plugins/readers/_shared.py index c31441f..f76accc 100644 --- a/plugins/atshosts/_shared.py +++ b/plugins/readers/_shared.py @@ -3,7 +3,7 @@ handed, and the date, URL and salary helpers more than one host needs. Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both import it without either importing the other. The leading underscore keeps -`plugins.atshosts` from listing it as a host. +`plugins.readers` from listing it as a host. """ from __future__ import annotations @@ -16,7 +16,9 @@ from typing import Any, Protocol import httpx -#: What every parser turns a feed into: one dict per row, with the keys +#: What an undeclared module's parser turns a feed into; a module that +#: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`. +#: One dict per row, with the keys #: `source_id`, `title`, `location`, `url`, `posted`, `department`, #: `description`, `remote` and `salary_text`, and, where a host states pay, #: `salary_min`, `salary_max` and `currency`. diff --git a/plugins/atshosts/ashby.py b/plugins/readers/ashby.py similarity index 100% rename from plugins/atshosts/ashby.py rename to plugins/readers/ashby.py diff --git a/plugins/atshosts/bamboohr.py b/plugins/readers/bamboohr.py similarity index 100% rename from plugins/atshosts/bamboohr.py rename to plugins/readers/bamboohr.py diff --git a/plugins/atshosts/deel.py b/plugins/readers/deel.py similarity index 100% rename from plugins/atshosts/deel.py rename to plugins/readers/deel.py diff --git a/plugins/atshosts/eightfold.py b/plugins/readers/eightfold.py similarity index 100% rename from plugins/atshosts/eightfold.py rename to plugins/readers/eightfold.py diff --git a/plugins/atshosts/gem.py b/plugins/readers/gem.py similarity index 100% rename from plugins/atshosts/gem.py rename to plugins/readers/gem.py diff --git a/plugins/atshosts/greenhouse.py b/plugins/readers/greenhouse.py similarity index 100% rename from plugins/atshosts/greenhouse.py rename to plugins/readers/greenhouse.py diff --git a/plugins/atshosts/hibob.py b/plugins/readers/hibob.py similarity index 100% rename from plugins/atshosts/hibob.py rename to plugins/readers/hibob.py diff --git a/plugins/atshosts/hrmanager.py b/plugins/readers/hrmanager.py similarity index 100% rename from plugins/atshosts/hrmanager.py rename to plugins/readers/hrmanager.py diff --git a/plugins/atshosts/jibe.py b/plugins/readers/jibe.py similarity index 100% rename from plugins/atshosts/jibe.py rename to plugins/readers/jibe.py diff --git a/plugins/atshosts/jobylon.py b/plugins/readers/jobylon.py similarity index 100% rename from plugins/atshosts/jobylon.py rename to plugins/readers/jobylon.py diff --git a/plugins/atshosts/lever.py b/plugins/readers/lever.py similarity index 100% rename from plugins/atshosts/lever.py rename to plugins/readers/lever.py diff --git a/plugins/atshosts/oracle.py b/plugins/readers/oracle.py similarity index 100% rename from plugins/atshosts/oracle.py rename to plugins/readers/oracle.py diff --git a/plugins/atshosts/personio.py b/plugins/readers/personio.py similarity index 100% rename from plugins/atshosts/personio.py rename to plugins/readers/personio.py diff --git a/plugins/atshosts/phenom.py b/plugins/readers/phenom.py similarity index 100% rename from plugins/atshosts/phenom.py rename to plugins/readers/phenom.py diff --git a/plugins/atshosts/pinpoint.py b/plugins/readers/pinpoint.py similarity index 100% rename from plugins/atshosts/pinpoint.py rename to plugins/readers/pinpoint.py diff --git a/plugins/atshosts/recruitee.py b/plugins/readers/recruitee.py similarity index 100% rename from plugins/atshosts/recruitee.py rename to plugins/readers/recruitee.py diff --git a/plugins/atshosts/rippling.py b/plugins/readers/rippling.py similarity index 100% rename from plugins/atshosts/rippling.py rename to plugins/readers/rippling.py diff --git a/plugins/atshosts/smartrecruiters.py b/plugins/readers/smartrecruiters.py similarity index 100% rename from plugins/atshosts/smartrecruiters.py rename to plugins/readers/smartrecruiters.py diff --git a/plugins/atshosts/successfactors.py b/plugins/readers/successfactors.py similarity index 100% rename from plugins/atshosts/successfactors.py rename to plugins/readers/successfactors.py diff --git a/plugins/atshosts/teamtailor.py b/plugins/readers/teamtailor.py similarity index 100% rename from plugins/atshosts/teamtailor.py rename to plugins/readers/teamtailor.py diff --git a/plugins/atshosts/workable.py b/plugins/readers/workable.py similarity index 100% rename from plugins/atshosts/workable.py rename to plugins/readers/workable.py diff --git a/plugins/atshosts/workday.py b/plugins/readers/workday.py similarity index 100% rename from plugins/atshosts/workday.py rename to plugins/readers/workday.py diff --git a/pyproject.toml b/pyproject.toml index 08b71b6..5ecbbf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ ignore = [ "app/models.py" = ["RUF012"] "desk/models.py" = ["RUF012"] "desk/sources/spec.py" = ["RUF012"] +"plugins/readers/_contract.py" = ["RUF012"] # The boilerplate's own test, adapted here but kept in its style. "tests/test_coupling.py" = ["SIM102"] diff --git a/tests/test_ats.py b/tests/test_ats.py index 1a9ae27..7285d52 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -19,7 +19,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats # noqa: E402 -from plugins.atshosts import smartrecruiters # noqa: E402 +from plugins.readers import smartrecruiters # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py index c5f1170..be8ca53 100644 --- a/tests/test_ats_bamboohr.py +++ b/tests/test_ats_bamboohr.py @@ -17,8 +17,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs import discover # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import bamboohr # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import bamboohr # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -161,7 +161,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None: assert allow("configit.bamboohr.com") and allow("CONFIGIT.bamboohr.com") assert not allow("bamboohr.com") and not allow("other.bamboohr.com") assert not allow("configit.bamboohr.com.evil.example") and not allow("localhost") - assert atshosts.HOSTS["bamboohr"] is bamboohr and ats.HOSTS["bamboohr"] == "BambooHR" + assert readers.HOSTS["bamboohr"] is bamboohr and ats.HOSTS["bamboohr"] == "BambooHR" @pytest.mark.parametrize("slug", [ @@ -182,4 +182,4 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None: 'xy' 'z') assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")] - assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS} + assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} diff --git a/tests/test_ats_deel.py b/tests/test_ats_deel.py index 2d33524..26acff2 100644 --- a/tests/test_ats_deel.py +++ b/tests/test_ats_deel.py @@ -16,8 +16,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs.discover import boards_in # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import deel # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import deel # noqa: E402 PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes() URL = "https://jobs.deel.com/klarna" @@ -126,7 +126,7 @@ def test_a_page_that_cannot_be_read_fails_the_board(web: dict[str, Any]) -> None # ------------------------------------------------------- slugs and hosts def test_the_module_is_registered_as_a_feed_host() -> None: - assert atshosts.HOSTS["deel"] is deel and ats.HOSTS["deel"] == "Deel" + assert readers.HOSTS["deel"] is deel and ats.HOSTS["deel"] == "Deel" assert deel.NAME == "deel" and deel.KEEP_CASE assert not hasattr(deel, "page") and not hasattr(deel, "detail") diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py index 398639a..093a5d6 100644 --- a/tests/test_ats_eightfold.py +++ b/tests/test_ats_eightfold.py @@ -17,8 +17,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs import discover # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import eightfold # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import eightfold # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SLUG = "jobs.ericsson.com/ericsson.com" @@ -169,7 +169,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None: assert allow("jobs.ericsson.com") and allow("JOBS.ERICSSON.COM") assert not allow("ericsson.com") and not allow("evil.example") assert not allow("jobs.ericsson.com.evil.example") and not allow("localhost") - assert atshosts.HOSTS["eightfold"] is eightfold and ats.HOSTS["eightfold"] == "Eightfold" + assert readers.HOSTS["eightfold"] is eightfold and ats.HOSTS["eightfold"] == "Eightfold" @pytest.mark.parametrize("slug", [ @@ -192,4 +192,4 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None found = [(h, s) for h, s, _ in discover.boards_in(page)] assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found assert ("eightfold", "") in found - assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS} + assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py index d42e617..99cd1fe 100644 --- a/tests/test_ats_gem.py +++ b/tests/test_ats_gem.py @@ -17,8 +17,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs import discover # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import gem # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import gem # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" API = "https://jobs.gem.com/api/public/graphql" @@ -154,7 +154,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None: assert allow("jobs.gem.com") and allow("JOBS.gem.com") assert not allow("gem.com") and not allow("productboard.gem.com") assert not allow("jobs.gem.com.evil.example") and not allow("localhost") - assert atshosts.HOSTS["gem"] is gem and ats.HOSTS["gem"] == "Gem" + assert readers.HOSTS["gem"] is gem and ats.HOSTS["gem"] == "Gem" @pytest.mark.parametrize("slug", [ @@ -177,4 +177,4 @@ def test_discovery_finds_the_board_keeping_its_case() -> None: 'y') assert [(h, s) for h, s, _ in discover.boards_in(page)] == [ ("gem", "Productboard"), ("gem", "sennder-gmbh")] - assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS} + assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} diff --git a/tests/test_ats_hibob.py b/tests/test_ats_hibob.py index 2eaf9b8..9df2125 100644 --- a/tests/test_ats_hibob.py +++ b/tests/test_ats_hibob.py @@ -16,8 +16,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs.discover import boards_in # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import hibob # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import hibob # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "hibob.json").read_bytes() @@ -171,7 +171,7 @@ def test_the_robots_txt_that_is_the_apps_own_page_allows(web: dict[str, Any]) -> # ------------------------------------------------------- slugs and hosts def test_the_module_is_registered_as_a_paged_host() -> None: - assert atshosts.HOSTS["hibob"] is hibob and ats.HOSTS["hibob"] == "HiBob" + assert readers.HOSTS["hibob"] is hibob and ats.HOSTS["hibob"] == "HiBob" assert hibob.NAME == "hibob" and not getattr(hibob, "KEEP_CASE", False) assert hasattr(hibob, "page") and not hasattr(hibob, "detail") diff --git a/tests/test_ats_hrmanager.py b/tests/test_ats_hrmanager.py index c1015f0..99b38ff 100644 --- a/tests/test_ats_hrmanager.py +++ b/tests/test_ats_hrmanager.py @@ -15,8 +15,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs.discover import boards_in # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import hrmanager # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import hrmanager # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "hrmanager.xml").read_bytes() @@ -155,7 +155,7 @@ def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide( # ------------------------------------------------------- slugs and hosts def test_the_module_is_registered_as_a_paged_host() -> None: - assert atshosts.HOSTS["hrmanager"] is hrmanager and ats.HOSTS["hrmanager"] == "HR Manager" + assert readers.HOSTS["hrmanager"] is hrmanager and ats.HOSTS["hrmanager"] == "HR Manager" assert hrmanager.NAME == "hrmanager" and not getattr(hrmanager, "KEEP_CASE", False) assert hasattr(hrmanager, "page") and not hasattr(hrmanager, "detail") diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py index 0d91086..27d0863 100644 --- a/tests/test_ats_jibe.py +++ b/tests/test_ats_jibe.py @@ -17,8 +17,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs import discover # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import jibe # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import jibe # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SITE = "jobs.booking.com" @@ -152,7 +152,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None: assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM") assert not allow("booking.com") and not allow("evil.example") assert not allow("jobs.booking.com.evil.example") and not allow("localhost") - assert atshosts.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe" + assert readers.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe" @pytest.mark.parametrize("slug", [ diff --git a/tests/test_ats_jobylon.py b/tests/test_ats_jobylon.py index 0133fb2..5a9b4e0 100644 --- a/tests/test_ats_jobylon.py +++ b/tests/test_ats_jobylon.py @@ -16,8 +16,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs.discover import boards_in # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import jobylon # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import jobylon # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "jobylon.html").read_bytes() @@ -158,7 +158,7 @@ def test_an_empty_board_is_complete(web: dict[str, Any]) -> None: # ------------------------------------------------------- slugs and hosts def test_the_module_is_registered_as_a_feed_host_with_detail() -> None: - assert atshosts.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon" + assert readers.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon" assert jobylon.NAME == "jobylon" and not getattr(jobylon, "KEEP_CASE", False) assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page") diff --git a/tests/test_ats_oracle.py b/tests/test_ats_oracle.py index 6a3591e..91e17f5 100644 --- a/tests/test_ats_oracle.py +++ b/tests/test_ats_oracle.py @@ -20,8 +20,8 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats # noqa: E402 -from plugins.atshosts import oracle # noqa: E402 -from plugins.atshosts._shared import is_web_url # noqa: E402 +from plugins.readers import oracle # noqa: E402 +from plugins.readers._shared import is_web_url # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" HOST = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com" diff --git a/tests/test_ats_phenom.py b/tests/test_ats_phenom.py index a5ce596..9ea73d5 100644 --- a/tests/test_ats_phenom.py +++ b/tests/test_ats_phenom.py @@ -19,8 +19,8 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats # noqa: E402 -from plugins.atshosts import phenom # noqa: E402 -from plugins.atshosts._shared import is_web_url # noqa: E402 +from plugins.readers import phenom # noqa: E402 +from plugins.readers._shared import is_web_url # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SLUG = "careers.king.com/us/en" diff --git a/tests/test_ats_pinpoint.py b/tests/test_ats_pinpoint.py index f1547d2..59d0ba0 100644 --- a/tests/test_ats_pinpoint.py +++ b/tests/test_ats_pinpoint.py @@ -14,8 +14,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs import discover # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import pinpoint # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import pinpoint # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -122,7 +122,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None: assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com") assert not allow("pinpointhq.com") and not allow("other.pinpointhq.com") assert not allow("xeneta.pinpointhq.com.evil.example") and not allow("localhost") - assert atshosts.HOSTS["pinpoint"] is pinpoint and ats.HOSTS["pinpoint"] == "Pinpoint" + assert readers.HOSTS["pinpoint"] is pinpoint and ats.HOSTS["pinpoint"] == "Pinpoint" @pytest.mark.parametrize("slug", [ @@ -146,7 +146,7 @@ def test_discovery_finds_the_subdomain() -> None: '') assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")] assert discover.boards_in("https://pinpointhq.com/") == [] - assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS} + assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} def test_a_pinpoint_board_is_not_probed() -> None: diff --git a/tests/test_ats_rippling.py b/tests/test_ats_rippling.py index a60780f..54b94dc 100644 --- a/tests/test_ats_rippling.py +++ b/tests/test_ats_rippling.py @@ -17,8 +17,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs.discover import boards_in # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import rippling # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import rippling # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "rippling.json").read_bytes() @@ -168,7 +168,7 @@ def test_robots_are_read_from_the_site_and_disallow_only_internal( # ------------------------------------------------------- slugs and hosts def test_the_module_is_registered_as_a_feed_host_with_detail() -> None: - assert atshosts.HOSTS["rippling"] is rippling and ats.HOSTS["rippling"] == "Rippling" + assert readers.HOSTS["rippling"] is rippling and ats.HOSTS["rippling"] == "Rippling" assert rippling.NAME == "rippling" and rippling.KEEP_CASE assert hasattr(rippling, "detail") and not hasattr(rippling, "page") diff --git a/tests/test_ats_successfactors.py b/tests/test_ats_successfactors.py index 97cf4fa..df84b6e 100644 --- a/tests/test_ats_successfactors.py +++ b/tests/test_ats_successfactors.py @@ -1,4 +1,4 @@ -"""The SuccessFactors host: `plugins/atshosts/successfactors.py` against saved +"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved career-site pages. No network. The fixtures are trimmed pages of Vestas and SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site with a path prefix), and the job pages of Vestas (`addressCountry`), @@ -19,8 +19,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.jobs.discover import boards_in # noqa: E402 -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import successfactors # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import successfactors # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -439,7 +439,7 @@ def test_careers_url_and_robots_page_are_the_search_page() -> None: def test_the_module_is_case_sensitive_about_the_prefix() -> None: assert successfactors.KEEP_CASE is True - assert atshosts.HOSTS["successfactors"] is successfactors + assert readers.HOSTS["successfactors"] is successfactors assert successfactors._parts("jobs.teradyne.com/MiR") == ("jobs.teradyne.com", "/MiR") diff --git a/tests/test_ats_workday.py b/tests/test_ats_workday.py index ecdff47..8a6cfaa 100644 --- a/tests/test_ats_workday.py +++ b/tests/test_ats_workday.py @@ -20,7 +20,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats # noqa: E402 -from plugins.atshosts import workday # noqa: E402 +from plugins.readers import workday # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SLUG = "zendesk.wd1/zendesk" diff --git a/tests/test_atshosts.py b/tests/test_readers.py similarity index 96% rename from tests/test_atshosts.py rename to tests/test_readers.py index a5eb53c..fe1ccd8 100644 --- a/tests/test_atshosts.py +++ b/tests/test_readers.py @@ -1,4 +1,4 @@ -"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for +"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for every host: the registry and its contract check, the fetch a host is handed, the per-role detail reads, robots.txt, and Lever's EU instance. No network — a fake host stands in where a behaviour belongs to no real host yet.""" @@ -22,8 +22,8 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from plugins import ats, atshosts # noqa: E402 -from plugins.atshosts import lever # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import lever # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -31,14 +31,15 @@ FIXTURES = ROOT / "tests" / "fixtures" / "ats" # --------------------------------------------------------------- registry def test_the_registry_finds_every_module() -> None: - names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py") + names = sorted(p.stem for p in (ROOT / "plugins" / "readers").glob("*.py") if not p.stem.startswith("_")) - assert sorted(atshosts.HOSTS) == names + assert sorted(readers.READERS) == names + assert set(readers.HOSTS) | set(readers.PROVIDERS) == set(names) assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"} - assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items()) + assert all(not readers.problems(module, name) for name, module in readers.READERS.items()) # The Sources page's select and the plugin's description follow the modules. - assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS + assert {name: module.LABEL for name, module in readers.HOSTS.items()} == ats.HOSTS assert "Teamtailor" in ats.ATSSource.description @@ -86,7 +87,7 @@ def test_the_registry_loads_a_package_by_name_and_skips_private_modules( path = _package(tmp_path, monkeypatch, "hostsok", { "zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"), "_helpers": "X = 1\n"}) - assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"] + assert list(readers.load(path, "hostsok")) == ["alpha", "zeta"] def test_the_registry_orders_by_rank_then_name( @@ -95,7 +96,7 @@ def test_the_registry_orders_by_rank_then_name( "alpha": _GOOD.format(name="alpha"), # the default, 100 "beta": _GOOD.format(name="beta") + "RANK = 5\n", "gamma": _GOOD.format(name="gamma") + "RANK = 100\n"}) - assert list(atshosts.load(path, "hostsranked")) == ["beta", "alpha", "gamma"] + assert list(readers.load(path, "hostsranked")) == ["beta", "alpha", "gamma"] def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None: @@ -107,8 +108,8 @@ def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None: assert ats.ATSSource.description.startswith( "Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, " "Recruitee, Personio, Teamtailor") - assert all(module.RANK < atshosts.DEFAULT_RANK - for name, module in atshosts.HOSTS.items() if name in first) + assert all(module.RANK < readers.DEFAULT_RANK + for name, module in readers.HOSTS.items() if name in first) def test_the_registry_refuses_a_module_missing_a_required_name( @@ -116,7 +117,7 @@ def test_the_registry_refuses_a_module_missing_a_required_name( broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url") path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken}) with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"): - atshosts.load(path, "hostsbad") + readers.load(path, "hostsbad") def test_the_contract_check_names_each_problem() -> None: @@ -126,7 +127,7 @@ def test_the_contract_check_names_each_problem() -> None: module.page = lambda get, slug, cursor: None module.probe_ok = lambda body: True module.RANK, module.KEEP_CASE = "1", "yes" - found = atshosts.problems(module, "odd") + found = readers.problems(module, "odd") assert "NAME is 'other', not the module's name 'odd'" in found assert "LABEL is not a non-empty string" in found assert any(p.startswith("PATTERNS has '('") for p in found) @@ -294,7 +295,7 @@ def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType, state: dict[str, str] | None = None) -> tuple[list[Any], Any]: - monkeypatch.setitem(atshosts.HOSTS, "fake", module) + monkeypatch.setitem(readers.HOSTS, "fake", module) source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) if state is not None: source.state = state @@ -411,7 +412,7 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list( base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["2"]'}) monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: ( - atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False)) + readers.HOSTS["fake"].parse(b'["2"]', "acme"), False)) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] @@ -422,7 +423,7 @@ def test_fetch_board_reads_the_listing_only( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"}) - monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) + monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail)) rows, complete = ats.fetch_board("fake:acme") assert ([r["description"] for r in rows], complete) == (["teaser"], True) assert web["asked"] == [f"{base}/list"] @@ -534,7 +535,7 @@ def _paged_host() -> types.ModuleType: module.page = page module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug) - assert not atshosts.problems(module, "fake") + assert not readers.problems(module, "fake") return module @@ -574,7 +575,7 @@ def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids( stored, is read again on the next pull.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"}) - monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) + monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail)) source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) roles: Any = source.fetch() # a generator, which `close` stops assert next(roles).body == "one" @@ -728,7 +729,7 @@ def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]: def test_a_robots_txt_redirect_to_the_www_twin_is_followed( monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True)) with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ pytest.raises(ats._Disallowed): ats._check_robots(client, "fake:acme", {}) @@ -740,13 +741,13 @@ def test_a_robots_txt_redirect_to_the_www_twin_is_followed( ]) def test_a_robots_txt_redirect_anywhere_else_is_unreachable( monkeypatch: pytest.MonkeyPatch, location: str) -> None: - monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True)) with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable): ats._check_robots(client, "fake:acme", {}) def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True)) with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ pytest.raises(ats._Refused): ats._get(client, ROBOTS, "fake", "acme") @@ -756,7 +757,7 @@ def test_fetch_board_honours_robots_txt( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", "https://jobs.fake.example/acme/list": b'["1"]'}) - monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True)) with pytest.raises(ats._Disallowed) as caught: ats.fetch_board("fake:acme") assert ats.failure_reason(caught.value) == "robots.txt disallows it" diff --git a/tests/test_readers_contract.py b/tests/test_readers_contract.py new file mode 100644 index 0000000..da2c8ee --- /dev/null +++ b/tests/test_readers_contract.py @@ -0,0 +1,387 @@ +"""The reader contract, held for every module of `plugins/readers/`. + +Per module: it declares `READS`, the declaration matches its functions, and +its rows keep what the contract says they keep. No network: fixtures stand +in for the host. + +`UNDECLARED` lists the modules that do not declare `READS` yet; their tests +are marked xfail (not strict), and the planner removes a name once the +module declares. A module not in the list must pass. + +Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/` +for searches, named after the module: + +- `-places.`: one listing answer with at least one role at two + or more places. For `listing="whole"` it is the body `parse` gets; for + `listing="paged"` it is what every request of the first `page` call + returns, and `rows` reads that page. Required by the test of a module + that declares `places="many"`, which is skipped until the fixture exists. +- `-places-detail.`: optional, for a host whose places come from + the role's own page: what `detail`'s requests return; its `places` replace + the row's. +- `-teaser.`: one listing answer with a teaser, read the same + way. Required by the test of a module with `text="excerpt"` or + `teaser=True`, skipped until it exists. +- `.slug`: optional, the slug the fixtures are read with, one line; + `acme` without it. +""" + +from __future__ import annotations + +import sys +import types +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pydantic +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import readers # noqa: E402 +from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402 +from plugins.readers import _contract # noqa: E402 + +FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" / "search") + +#: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and +#: C (the search providers, which arrive declared) land. +UNDECLARED = frozenset({ + "ashby", "bamboohr", "deel", "eightfold", "gem", "greenhouse", "hibob", "hrmanager", + "jibe", "jobylon", "lever", + "oracle", "personio", "phenom", "pinpoint", "recruitee", "rippling", "smartrecruiters", + "successfactors", "teamtailor", "workable", "workday", +}) + + +def _params() -> list[Any]: + return [pytest.param(name, marks=pytest.mark.xfail(reason="no READS yet", strict=False)) + if name in UNDECLARED else name for name in readers.READERS] + + +def _declared(name: str) -> tuple[types.ModuleType, Reads]: + module = readers.READERS[name] + reads = getattr(module, "READS", None) + assert isinstance(reads, Reads), f"{name} declares no READS" + return module, reads + + +def _fixture(name: str, kind: str) -> Path | None: + for folder in FIXTURE_DIRS: + found = sorted(folder.glob(f"{name}-{kind}.*")) + if found: + return found[0] + return None + + +def _slug(name: str) -> str: + for folder in FIXTURE_DIRS: + if (path := folder / f"{name}.slug").exists(): + return path.read_text(encoding="utf-8").strip() + return "acme" + + +def _answer(body: bytes) -> Callable[..., bytes]: + return lambda url, **kwargs: body + + +def read_fixture(module: types.ModuleType, body: bytes, slug: str, + detail_body: bytes | None = None) -> list[dict[str, Any]]: + """The rows `module` makes of one listing answer, as the runner reads + them: `parse` for a whole listing, the first `page` and its `rows` for a + paged one, and, given `detail_body`, each row's `places` from `detail`.""" + reads: Reads = module.READS + if reads.listing == "whole": + rows = list(module.parse(body, slug)) + else: + decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, Ask()) + rows = list(module.rows(decoded, slug)) + if detail_body is not None: + for row in rows: + found = module.detail(_answer(detail_body), slug, row) + if "places" in found: + row["places"] = found["places"] + return rows + + +def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]: + """What rows get wrong about `places` under the declared trait.""" + out = [] + for row in rows: + found = row.get("places") + rid = row.get("source_id") + if not isinstance(found, list): + out.append(f"{rid}: places is not a list") + continue + for place in found: + if not (isinstance(place, dict) and set(place) == {"text", "remote"} + and isinstance(place["text"], str) and place["text"].strip() + and place["remote"] in (True, False, None)): + out.append(f"{rid}: {place!r} is not a Place") + if places in ("one", "copies") and len(found) > 1: + out.append(f'{rid}: {len(found)} places under places="{places}"') + if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows): + out.append('no row has two places: a reader with places="many" keeps every place') + return out + + +# ------------------------------------------------------------ every module + +def test_the_undeclared_list_names_only_readers() -> None: + assert set(readers.READERS) >= UNDECLARED + + +def test_undeclared_lists_the_modules_without_reads() -> None: + assert set(readers.undeclared()) == {name for name, module in readers.READERS.items() + if not hasattr(module, "READS")} + + +@pytest.mark.parametrize("name", _params()) +def test_a_reader_declares_reads(name: str) -> None: + _declared(name) + + +@pytest.mark.parametrize("name", _params()) +def test_a_declaration_matches_the_functions(name: str) -> None: + module, _reads = _declared(name) + assert readers.problems(module, name) == [] + expected = "search" if module.READS.scope == "search" else "board" + assert name in (readers.PROVIDERS if expected == "search" else readers.HOSTS) + + +@pytest.mark.parametrize("name", _params()) +def test_a_reader_keeps_every_place(name: str) -> None: + module, reads = _declared(name) + fixture = _fixture(name, "places") + if fixture is None: + if reads.places == "many": + pytest.skip(f"no fixture {name}-places.* yet") + pytest.skip(f'places="{reads.places}" and no fixture') + detail = _fixture(name, "places-detail") + rows = read_fixture(module, fixture.read_bytes(), _slug(name), + detail.read_bytes() if detail else None) + assert rows, f"{fixture.name} gave no rows" + assert place_problems(rows, reads.places) == [] + + +@pytest.mark.parametrize("name", _params()) +def test_a_reader_keeps_the_teaser(name: str) -> None: + module, reads = _declared(name) + if not (reads.text == "excerpt" or reads.teaser): + pytest.skip("the list carries no teaser") + fixture = _fixture(name, "teaser") + if fixture is None: + pytest.skip(f"no fixture {name}-teaser.* yet") + rows = read_fixture(module, fixture.read_bytes(), _slug(name)) + assert any(isinstance(row.get("teaser"), str) and row["teaser"].strip() for row in rows) + + +# ------------------------------------------------------------ the checks + +def _reader(**functions: Any) -> Any: + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", (r"fake\.example/([a-z]+)",) + module.slug_ok = lambda slug: True + module.allowed = lambda slug: lambda host: host == "fake.example" + module.careers_url = lambda slug: "https://fake.example/" + slug + for name, function in functions.items(): + setattr(module, name, function) + return module + + +def _whole(**functions: Any) -> Any: + return _reader(**{"feed_url": lambda slug, ask: "https://fake.example/" + slug, + "parse": lambda body, slug: []} | functions) + + +def _paged(**functions: Any) -> Any: + return _reader(**{"page": lambda get, slug, cursor, ask: (None, [], None, None), + "rows": lambda decoded, slug: []} | functions) + + +_ROBOTS = "read: no robots.txt rule on /api/ (checked 2026-09-30)" + + +def _reads(**traits: Any) -> Reads: + base: dict[str, Any] = {"scope": "board", "listing": "whole", "text": "listing", + "complete": "whole", "places": "one", "robots": _ROBOTS} + return Reads(**(base | traits)) + + +def test_a_good_declaration_has_no_problems() -> None: + module = _whole() + module.READS = _reads() + assert readers.problems(module, "fake") == [] + module = _paged(detail=lambda get, slug, row: {}, + facets=lambda get, slug: {"country": {"DE": "1"}}) + module.READS = _reads(listing="paged", page_size=20, ceiling=2000, text="detail", + teaser=True, complete="stated", filters=Filters(country="ids"), + places="many", fields=Fields(posted="listed", remote="stated"), + empty="suspect", title="provisional") + assert readers.problems(module, "fake") == [] + + +def test_a_module_without_reads_is_checked_as_before() -> None: + module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: []) + assert readers.problems(module, "fake") == [] + assert readers.scope(module) == "board" + assert readers.undeclared({"fake": module}) == ["fake"] + + +@pytest.mark.parametrize(("module", "reads", "problem"), [ + (_reader(feed_url=lambda slug, ask: "", parse=lambda b, s: []), + {"listing": "paged", "complete": "stated"}, 'listing="paged" without page and rows'), + (_paged(), {}, 'listing="whole" without feed_url and parse'), + (_whole(), {"complete": "stated"}, 'complete="whole" goes with listing="whole"'), + (_whole(), {"page_size": 20}, 'page_size and ceiling are for listing="paged"'), + (_paged(), {"listing": "paged", "complete": "end", "page_size": 100, "ceiling": 50}, + "ceiling is below page_size"), + (_paged(fallback_url=lambda slug: ""), {"listing": "paged", "complete": "end"}, + 'fallback_url without listing="whole"'), + (_whole(), {"text": "detail"}, 'text="detail" without detail'), + (_whole(detail=lambda g, s, r: {}), {}, 'detail with text="listing"'), + (_whole(), {"teaser": True, "text": "excerpt"}, "teaser is for text=\"detail\""), + (_whole(), {"title": "provisional"}, 'title="provisional" without detail'), + (_whole(), {"filters": Filters(country="ids")}, "an ids filter (country) without facets"), + (_whole(facets=lambda g, s: {}), {"filters": Filters(country="one")}, + "facets without an ids filter"), + (_whole(facets=lambda g, s: {}), {"filters": Filters(query="ids")}, + 'query cannot be "ids"'), + (_paged(), {"scope": "search", "listing": "paged", "complete": "end"}, + 'scope="search" needs complete="never" or "stated"'), + (_paged(), {"listing": "paged", "complete": "never"}, + 'scope="board" cannot be complete="never"'), + (_whole(), {"key": Key(name="Adzuna Key", fields=("app_id",))}, + "key names no config entry"), + (_whole(), {"key": Key(name="adzuna", fields=())}, "key has no fields"), + (_whole(), {"key": Key(name="adzuna", fields=("app_id", "app_key"), + env=("ADZUNA_APP_ID",))}, + "key.env does not match key.fields"), + (_whole(), {"robots": " "}, "robots is empty"), + (_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {}, + "feed_url does not take (slug, ask)"), + (_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []), + {"listing": "paged", "complete": "end"}, "page does not take (get, slug, cursor, ask)"), +]) +def test_the_declaration_check_names_each_problem( + module: Any, reads: dict[str, Any], problem: str) -> None: + module.READS = _reads(**reads) + found = readers.problems(module, "fake") + assert any(p.startswith(problem) for p in found), found + + +def test_reads_that_is_not_a_reads_is_refused() -> None: + module = _whole() + module.READS = {"scope": "board"} + assert "READS is not a Reads" in readers.problems(module, "fake") + + +def test_a_search_needs_fewer_names_than_a_board() -> None: + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL = "fake", "Fake" + module.allowed = lambda slug: lambda host: True + module.page = lambda get, slug, cursor, ask: (None, [], 0, None) + module.rows = lambda decoded, slug: [] + module.READS = _reads(scope="search", listing="paged", complete="never", + filters=Filters(query="one"), + key=Key(name="adzuna", fields=("app_id", "app_key"), + env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY"))) + assert readers.problems(module, "fake") == [] + assert readers.scope(module) == "search" + del module.READS + assert "no careers_url" in readers.problems(module, "fake") + + +def test_a_misspelt_trait_fails_where_it_is_declared() -> None: + with pytest.raises(pydantic.ValidationError): + _reads(listing="pages") + with pytest.raises(pydantic.ValidationError): + Filters(country="some") + with pytest.raises(pydantic.ValidationError): + _reads(robots=None) + with pytest.raises(pydantic.ValidationError): + Reads(scope="board", listing="whole", text="listing", complete="whole", + places="one") # type: ignore[call-arg] # robots is required + with pytest.raises(pydantic.ValidationError): + _reads(colour="blue") # extra keys are refused + + +def test_a_declaration_is_frozen() -> None: + reads = _reads() + with pytest.raises(pydantic.ValidationError): + reads.places = "many" # type: ignore[misc] + + +def test_the_loader_refuses_a_module_whose_declaration_breaks_the_contract( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + package = tmp_path / "readersbad" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "odd.py").write_text( + "from plugins.readers import Reads\n" + "NAME = 'odd'\nLABEL = 'Odd'\nPATTERNS = ()\n" + "def slug_ok(slug): return True\n" + "def allowed(slug): return lambda h: True\n" + "def careers_url(slug): return ''\n" + "def feed_url(slug, ask): return ''\n" + "def parse(body, slug): return []\n" + "READS = Reads(scope='board', listing='whole', text='detail', complete='whole',\n" + " places='many', robots='read (checked 2026-09-30)')\n") + monkeypatch.syspath_prepend(str(tmp_path)) + with pytest.raises(ImportError, match=r'readersbad\.odd .*text="detail" without detail'): + readers.load([str(package)], "readersbad") + + +def test_filters_list_what_is_filtered() -> None: + assert Filters(country="one", query="many").declared() == {"country": "one", + "query": "many"} + + +# ------------------------------------------------------------ the fixture check + +def _fake_many(rows: list[dict[str, Any]]) -> Any: + module = _reader(feed_url=lambda slug, ask: "", parse=lambda body, slug: rows) + module.READS = _reads(places="many") + return module + + +def test_the_place_check_passes_a_role_at_two_places() -> None: + rows = [{"source_id": "1", "places": [{"text": "Berlin", "remote": False}, + {"text": "Remote (DE)", "remote": True}]}] + assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == [] + + +def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None: + joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}] + assert place_problems(joined, "many") == [ + 'no row has two places: a reader with places="many" keeps every place'] + assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [ + "2: places is not a list"] + assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [ + "3: ('Berlin', False) is not a Place"] + + +def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None: + seen: list[bytes] = [] + + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + seen.append(get("https://fake.example/list")) + return ([{"id": "1"}], [{"id": "1"}], 1, None) + + module = _paged(page=page, rows=lambda decoded, slug: [{"source_id": d["id"], "places": []} + for d in decoded], + detail=lambda get, slug, row: {"places": [ + {"text": get("https://fake.example/1").decode(), "remote": None}, + {"text": "Munich", "remote": None}]}) + module.READS = _reads(listing="paged", complete="stated", text="detail", places="many") + rows = read_fixture(module, b"Berlin", "acme", detail_body=b"Berlin") + assert seen == [b"Berlin"] + assert place_problems(rows, "many") == [] + + +def test_the_contract_types_are_what_the_package_exports() -> None: + assert readers.Reads is _contract.Reads and readers.Row is _contract.Row + assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}