Merge branch 'wp-a-contract'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 08:56:58 +02:00
commit f4b0ffc28f
52 files changed
+1115 -321

No files matched your search

+1 -1
View File
@@ -1,6 +1,6 @@
# roledesk # roledesk
A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor, Workday, SuccessFactors, Oracle, Phenom, Eightfold and a dozen smaller hosts, one module each in `plugins/atshosts/`) and runs saved searches on boards that span companies (Jobindex, Adzuna), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make. A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor, Workday, SuccessFactors, Oracle, Phenom, Eightfold and a dozen smaller hosts, one module each in `plugins/readers/`) and runs saved searches on boards that span companies (Jobindex, Adzuna), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make.
Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeline/`), password (`auth/`) and deploy kit (`deploy/`), plus the plugin contract and tools from news (`desk/`). FastAPI, Jinja2, SQLite, `uv`, no build step. Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeline/`), password (`auth/`) and deploy kit (`deploy/`), plus the plugin contract and tools from news (`desk/`). FastAPI, Jinja2, SQLite, `uv`, no build step.
+8 -8
View File
@@ -5,7 +5,7 @@ The seed list and the LinkedIn flow both start from a company name and a
website, not from a board. This module gets from the one to the other: it website, not from a board. This module gets from the one to the other: it
fetches the site's careers page, or the pages the home page links to as fetches the site's careers page, or the pages the home page links to as
careers, and looks for a link into one of the hosts in `HOSTS`. A readable careers, and looks for a link into one of the hosts in `HOSTS`. A readable
host (one with a module in `plugins/atshosts/`, which the `ats` plugin host (one with a module in `plugins/readers/`, which the `ats` plugin
parses) becomes a `board` target the pull can read; any other host is parses) becomes a `board` target the pull can read; any other host is
recorded as `ats`, with its slug where the URL carries one, so the company is recorded as `ats`, with its slug where the URL carries one, so the company is
kept as one the app cannot read yet rather than dropped. The patterns and kept as one the app cannot read yet rather than dropped. The patterns and
@@ -48,7 +48,7 @@ from urllib.parse import urljoin
import httpx import httpx
from desk.sources import USER_AGENT from desk.sources import USER_AGENT
from plugins import atshosts from plugins import readers
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -58,7 +58,7 @@ MAX_BYTES = 2 << 20
MAX_PAGES = 6 MAX_PAGES = 6
#: The hosts the `ats` plugin parses; a match on one of these is a `board`. #: The hosts the `ats` plugin parses; a match on one of these is a `board`.
READABLE: tuple[str, ...] = tuple(atshosts.HOSTS) READABLE: tuple[str, ...] = tuple(readers.HOSTS)
#: host -> pattern over a URL, for the hosts discovery names but the app #: host -> pattern over a URL, for the hosts discovery names but the app
#: cannot read. Group 1, where there is one, is the slug the host names the #: cannot read. Group 1, where there is one, is the slug the host names the
@@ -104,14 +104,14 @@ def _hosts(modules: Mapping[str, ModuleType],
#: host -> pattern over a URL, every host discovery can name. The slug is #: host -> pattern over a URL, every host discovery can name. The slug is
#: group 1, or, for a pattern with more groups, the groups that matched #: group 1, or, for a pattern with more groups, the groups that matched
#: joined by "/" (Lever's `eu/<site>`). #: joined by "/" (Lever's `eu/<site>`).
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE) HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(readers.HOSTS, _UNREADABLE)
#: The hosts whose slugs keep the case the URL wrote them in, by their #: The hosts whose slugs keep the case the URL wrote them in, by their
#: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its #: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its
#: listing and its robots.txt paths are case-sensitive. Every other slug is #: listing and its robots.txt paths are case-sensitive. Every other slug is
#: lowercased, which is how a page that writes `BoschGroup` and one that #: lowercased, which is how a page that writes `BoschGroup` and one that
#: writes `boschgroup` name one board. #: writes `boschgroup` name one board.
_KEEP_CASE = frozenset(name for name, module in atshosts.HOSTS.items() _KEEP_CASE = frozenset(name for name, module in readers.HOSTS.items()
if getattr(module, "KEEP_CASE", False)) if getattr(module, "KEEP_CASE", False))
#: What a Teamtailor feed carries and a page that only looks like one does not: #: What a Teamtailor feed carries and a page that only looks like one does not:
@@ -257,14 +257,14 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
#: rate limit that a run of guesses trips, and a Workable site links its #: rate limit that a run of guesses trips, and a Workable site links its
#: board plainly anyway. #: board plainly anyway.
_PROBES: dict[str, Callable[[str], str]] = { _PROBES: dict[str, Callable[[str], str]] = {
name: module.probe_url for name, module in atshosts.HOSTS.items() name: module.probe_url for name, module in readers.HOSTS.items()
if hasattr(module, "probe_url")} if hasattr(module, "probe_url")}
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]: def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
"""The slugs a guessed name is probed as on `host`: the name itself, and """The slugs a guessed name is probed as on `host`: the name itself, and
for Lever the name on its EU instance too (the module's `probe_slugs`).""" for Lever the name on its EU instance too (the module's `probe_slugs`)."""
module = atshosts.HOSTS[host] module = readers.HOSTS[host]
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,) return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
@@ -281,7 +281,7 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool:
return False return False
if not response.is_success: if not response.is_success:
return False return False
module = atshosts.HOSTS[host] module = readers.HOSTS[host]
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
+1 -1
View File
@@ -663,7 +663,7 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
#: read as when a role was posted, checked against live boards on #: read as when a role was posted, checked against live boards on
#: 2026-09-29: #: 2026-09-29:
#: #:
#: - `greenhouse`: `plugins.atshosts.greenhouse.parse` already prefers #: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers
#: `first_published` over `updated_at`. A live board (adyen) showed a job #: `first_published` over `updated_at`. A live board (adyen) showed a job
#: with `first_published` five months before `updated_at`, proving the #: with `first_published` five months before `updated_at`, proving the
#: first stays fixed while the second moves on an edit — the resettable #: first stays fixed while the second moves on an edit — the resettable
+1 -1
View File
@@ -100,7 +100,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**A board that fails on every fetch for 21 days closes its roles.** A board whose fetch raised was only logged, so a company that moved ATS kept its roles open for good behind a 404. The `ats` plugin now records `ats:failing:<target>` in `state` with the time of the first failure in a row and the latest reason, and a fetch that answers removes it. Once the first failure is `STALE_DAYS` old, the same window a search role gets, the settle closes the board's roles; the next answer reopens what it lists through `upsert_role`, as for a board switched back on. The record is plugin state rather than a column because the plugin only sees `self.state`. `/api/companies` returns it as `failing_since` and `failing_reason`, the Companies list shows "not answering since", and `app companies check` prints it beside its live result. **A board that fails on every fetch for 21 days closes its roles.** A board whose fetch raised was only logged, so a company that moved ATS kept its roles open for good behind a 404. The `ats` plugin now records `ats:failing:<target>` in `state` with the time of the first failure in a row and the latest reason, and a fetch that answers removes it. Once the first failure is `STALE_DAYS` old, the same window a search role gets, the settle closes the board's roles; the next answer reopens what it lists through `upsert_role`, as for a board switched back on. The record is plugin state rather than a column because the plugin only sees `self.state`. `/api/companies` returns it as `failing_since` and `failing_reason`, the Companies list shows "not answering since", and `app companies check` prints it beside its live result.
**A host is a module in `plugins/atshosts/`, and registers by existing.** `plugins/ats.py` held eight hosts in six parallel tables and `app/jobs/discover.py` three more, so two people adding a host edited the same lines. Each host is now one file that states its name, label, URL patterns, slug check, allowed host names and either a feed and a parser or a pager; `ats.py` and `discover.py` build their tables from the package, ordered by the module's `RANK` so the select and the probes kept their order. The package sits beside `ats.py` and is not `plugins/ats/`, since `desk.sources.registry` globs `plugins/*.py` and the tests patch names on the `ats` module. The contract is the docstring of `plugins/atshosts/__init__.py`, and a module that breaks it fails the import by name. **A host is a module in `plugins/readers/`, and registers by existing.** `plugins/ats.py` held eight hosts in six parallel tables and `app/jobs/discover.py` three more, so two people adding a host edited the same lines. Each host is now one file that states its name, label, URL patterns, slug check, allowed host names and either a feed and a parser or a pager; `ats.py` and `discover.py` build their tables from the package, ordered by the module's `RANK` so the select and the probes kept their order. The package sits beside `ats.py` and is not `plugins/ats/`, since `desk.sources.registry` globs `plugins/*.py` and the tests patch names on the `ats` module. The contract is the docstring of `plugins/readers/__init__.py`, and a module that breaks it fails the import by name.
**A host is handed the fetch.** A module imports `desk` and `._shared` only and receives `get`, which `ats.py` binds to its own `_get` at call time. The allow-list, the redirect rule, the byte cap and the deadline stay in one place, and a test that patches `ats._get` reaches every host. `_get` refuses an IP address, a local or numeric host name and any port but 443 before it asks, because a Teamtailor or SuccessFactors slug is a host name somebody typed. **A host is handed the fetch.** A module imports `desk` and `._shared` only and receives `get`, which `ats.py` binds to its own `_get` at call time. The allow-list, the redirect rule, the byte cap and the deadline stay in one place, and a test that patches `ats._get` reaches every host. `_get` refuses an IP address, a local or numeric host name and any port but 443 before it asks, because a Teamtailor or SuccessFactors slug is a host name somebody typed.
+2 -2
View File
@@ -28,11 +28,11 @@ The survey's record of each company is in `docs/plan-hosts/survey-own.json` (cla
## Work packages ## Work packages
Each owns its modules in `plugins/atshosts/`, `tests/test_ats_<name>.py` and `tests/fixtures/ats/<name>*`, and nothing else. The common rules of wave 1 apply: the live request limits, robots.txt read first, no key from a script, a final report with the targets to attach. Each owns its modules in `plugins/readers/`, `tests/test_ats_<name>.py` and `tests/fixtures/ats/<name>*`, and nothing else. The common rules of wave 1 apply: the live request limits, robots.txt read first, no key from a script, a final report with the targets to attach.
| WP | Model | Builds | Needs | | WP | Model | Builds | Needs |
|---|---|---|---| |---|---|---|---|
| 9 Provisional title | Opus | the row key, `upsert_role`'s rule and its tests; owns `plugins/ats.py`, `plugins/atshosts/__init__.py`, `app/storage/store.py`, `tests/test_atshosts.py`, `tests/test_storage.py` | the first decision | | 9 Provisional title | Opus | the row key, `upsert_role`'s rule and its tests; owns `plugins/ats.py`, `plugins/readers/__init__.py`, `app/storage/store.py`, `tests/test_readers.py`, `tests/test_storage.py` | the first decision |
| 10 Sitemap reader | Sonnet | `sitemap`: job URLs from a sitemap, the role from the page's JSON-LD or microdata | WP9 | | 10 Sitemap reader | Sonnet | `sitemap`: job URLs from a sitemap, the role from the page's JSON-LD or microdata | WP9 |
| 11 Board filters | Opus | `params` on a board subscription, handed to `page`; owns `plugins/ats.py`, `app/jobs/pull.py`, `app/jobs/sources.py`, `app/web/api.py`, `app/static/companies.js` and their tests | the second decision | | 11 Board filters | Opus | `params` on a board subscription, handed to `page`; owns `plugins/ats.py`, `app/jobs/pull.py`, `app/jobs/sources.py`, `app/web/api.py`, `app/static/companies.js` and their tests | the second decision |
| 12 Own APIs | Sonnet | Ørsted, Amazon, Vinted, Dynatrace, RELEX, Siteimprove, Bending Spoons | | | 12 Own APIs | Sonnet | Ørsted, Amazon, Vinted, Dynatrace, RELEX, Siteimprove, Bending Spoons | |
+2 -2
View File
@@ -1,8 +1,8 @@
# Where roles come from # Where roles come from
This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a host module is the docstring of `plugins/atshosts/__init__.py`, not this file. This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a reader (a host module or a search provider) is the docstring of `plugins/readers/__init__.py`, with its types and the declaration check in `plugins/readers/_contract.py` and the conformance test in `tests/test_readers_contract.py`, not this file.
The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/atshosts/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`. The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/readers/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
## The three ways a source is read ## The three ways a source is read
+2 -2
View File
@@ -4,7 +4,7 @@ A plugin file imports `desk` only — see `tests/test_architecture.py` — and
its own helpers in this directory. This `__init__.py` makes the directory its own helpers in this directory. This `__init__.py` makes the directory
importable as `plugins.<name>`, for the rest of roledesk (`jobs/pull.py`, importable as `plugins.<name>`, for the rest of roledesk (`jobs/pull.py`,
`cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the `cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the
package `plugins/atshosts/`. `desk.sources.registry` still finds `ats.py` by package `plugins/readers/`. `desk.sources.registry` still finds `ats.py` by
path, not by this package; `ats.py` puts the repo root on `sys.path` itself path, not by this package; `ats.py` puts the repo root on `sys.path` itself
when the app has not, so that `plugins.atshosts` imports either way. when the app has not, so that `plugins.readers` imports either way.
""" """
+19 -19
View File
@@ -1,5 +1,5 @@
"""ATS boards: a company's own careers page on one of the hosts in """ATS boards: a company's own careers page on one of the hosts in
`plugins/atshosts/`, one module per host. `plugins/readers/`, one module per host.
Every one of these hosts serves a company's own list of roles as public, Every one of these hosts serves a company's own list of roles as public,
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
@@ -49,7 +49,7 @@ from desk.sources import (
) )
try: try:
from plugins import atshosts from plugins import readers
except ModuleNotFoundError: except ModuleNotFoundError:
# `desk.sources.registry` loads this file by its path, under a name of its # `desk.sources.registry` loads this file by its path, under a name of its
# own, and the app puts the repo root on `sys.path` before it does; a # own, and the app puts the repo root on `sys.path` before it does; a
@@ -57,16 +57,16 @@ except ModuleNotFoundError:
# does not, and the host modules are found as the package beside this # does not, and the host modules are found as the package beside this
# file. So the root goes on the path then, and only then. # file. So the root goes on the path then, and only then.
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import atshosts from plugins import readers
from plugins.atshosts._shared import Get, Row from plugins.readers._shared import Get, Row
# The names the parsers had here before they moved to their host modules, # The names the parsers had here before they moved to their host modules,
# kept for the callers and the tests that still reach for them on `ats`. # kept for the callers and the tests that still reach for them on `ats`.
from plugins.atshosts._shared import ( # noqa: F401 from plugins.readers._shared import ( # noqa: F401
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount, Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
) )
from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401 from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401
from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401 from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -101,7 +101,7 @@ DEADLINE = 60.0
#: host -> the name the host gives itself: the hosts this plugin reads, in #: host -> the name the host gives itself: the hosts this plugin reads, in
#: the order a select offers them. What the Sources page's host select is #: the order a select offers them. What the Sources page's host select is
#: rendered from, so it cannot offer a host `is_known_target` refuses. #: rendered from, so it cannot offer a host `is_known_target` refuses.
HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()} HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
@@ -110,7 +110,7 @@ def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
company's own site. Anything else is refused, at the first hop and at company's own site. Anything else is refused, at the first hop and at
every redirect: a feed is external content, and a board row must never every redirect: a feed is external content, and a board row must never
be able to point this fetch at an internal address.""" be able to point this fetch at an internal address."""
module = atshosts.HOSTS.get(host) module = readers.HOSTS.get(host)
return module.allowed(slug) if module is not None else lambda h: False return module.allowed(slug) if module is not None else lambda h: False
@@ -234,7 +234,7 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get:
#: host -> parser, for the hosts whose module parses a whole body. Every one #: host -> parser, for the hosts whose module parses a whole body. Every one
#: takes the raw response body and the slug. #: takes the raw response body and the slug.
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")} name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
# --------------------------------------------------------- the classifier # --------------------------------------------------------- the classifier
@@ -365,11 +365,11 @@ def classify(department: str, title: str) -> str:
def is_known_target(target: str) -> bool: def is_known_target(target: str) -> bool:
"""`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's """`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's
`slug_ok` accepts. What the target validator refuses: `app companies add` `slug_ok` accepts. What the target validator refuses: `app companies add`
and the pull job both call this before touching the network.""" and the pull job both call this before touching the network."""
host, sep, slug = target.partition(":") host, sep, slug = target.partition(":")
module = atshosts.HOSTS.get(host) module = readers.HOSTS.get(host)
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug) return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
@@ -377,7 +377,7 @@ def careers_url(target: str) -> str:
"""The company's own careers page for `host:slug`, or "" for an unknown """The company's own careers page for `host:slug`, or "" for an unknown
host. What `app companies add` stores alongside the board.""" host. What `app companies add` stores alongside the board."""
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host) module = readers.HOSTS.get(host)
return module.careers_url(slug) if module is not None else "" return module.careers_url(slug) if module is not None else ""
@@ -404,7 +404,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
A 429 after the first page stops the read with what it has, incomplete; A 429 after the first page stops the read with what it has, incomplete;
one on the first page raises like any other failure.""" one on the first page raises like any other failure."""
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
module = atshosts.HOSTS[host] module = readers.HOSTS[host]
get = _getter(client, host, slug) get = _getter(client, host, slug)
rows: dict[str, Row] = {} rows: dict[str, Row] = {}
seen: set[str] = set() seen: set[str] = set()
@@ -449,7 +449,7 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
if not is_known_target(target): if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host") raise ValueError(f"{target!r} is not host:slug for a known ATS host")
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
module = atshosts.HOSTS[host] module = readers.HOSTS[host]
if hasattr(module, "page"): if hasattr(module, "page"):
return _paged(client, target) return _paged(client, target)
get = _getter(client, host, slug) get = _getter(client, host, slug)
@@ -566,7 +566,7 @@ def _check_robots(client: httpx.Client, target: str,
kept, so the next pull asks again. A 429 counts as unreachable although kept, so the next pull asks again. A 429 counts as unreachable although
it is a 4xx, since it says "not now" rather than "there is none".""" it is a 4xx, since it says "not now" rather than "there is none"."""
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host) module = readers.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target): if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
return return
page = module.robots_page(slug) page = module.robots_page(slug)
@@ -657,7 +657,7 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
after a complete listing, so one cut short does not make the next pull after a complete listing, so one cut short does not make the next pull
read its missing roles again.""" read its missing roles again."""
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
module = atshosts.HOSTS[host] module = readers.HOSTS[host]
get = _getter(client, host, slug) get = _getter(client, host, slug)
done, failing = set(detailed), set(failed) - set(detailed) done, failing = set(detailed), set(failed) - set(detailed)
@@ -804,7 +804,7 @@ class ATSSource(Source):
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}") detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
detail_sets: tuple[set[str], set[str]] | None = None detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(atshosts.HOSTS.get(host), "detail"): if hasattr(readers.HOSTS.get(host), "detail"):
detailed, failed = (self._id_set(key) for key in detail_keys) detailed, failed = (self._id_set(key) for key in detail_keys)
rows, detailed, failed = _read_details(client, target, rows, complete, rows, detailed, failed = _read_details(client, target, rows, complete,
detailed, failed) detailed, failed)
@@ -833,7 +833,7 @@ class ATSSource(Source):
"function": classify(row["department"], row["title"]), "function": classify(row["department"], row["title"]),
"remote": row["remote"], "remote": row["remote"],
# Every parser sets salary_text, empty or not (see # Every parser sets salary_text, empty or not (see
# "salary_text" in `atshosts/_shared.py`); only # "salary_text" in `readers/_shared.py`); only
# the hosts that state structured pay also set the # the hosts that state structured pay also set the
# normalised figures, so the rest leave them at # normalised figures, so the rest leave them at
# row.get's default of no figure. # row.get's default of no figure.
-217
View File
@@ -1,217 +0,0 @@
"""The hosts the `ats` plugin reads, one module each.
A host registers by existing: every module in this package whose name does
not start with `_` is loaded when the package is imported, checked against
the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A
module that breaks the contract fails the import with an ImportError naming
the module and what it lacks, rather than a pull that fails board by board.
`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host
select and its description from `HOSTS`, and `app/jobs/discover.py` its
patterns and probes. The package sits beside `ats.py` rather than being
`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and
the tests patch names on the `ats` module.
A module is named after its host (`plugins/atshosts/workday.py`) and imports
`desk` and `._shared` only, never `plugins.ats`. It holds:
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
LABEL = "Workday" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int; lower comes first in the select and the probes
KEEP_CASE = False # optional; True keeps a discovered slug's case
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
def slug_ok(slug: str) -> bool # strict, see below
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
def careers_url(slug: str) -> str # the page a person opens
and one of two ways to read a board:
def feed_url(slug: str) -> str # the whole board in one answer,
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call,
def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page
and, optionally:
def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read
# once when `feed_url` is over the cap
def robots_page(slug: str) -> str # the URL robots.txt must allow
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page
def probe_url(slug: str) -> str # discovery: 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
hold 10 to 80, which keeps the order they had before there were modules, so
a new host comes after them unless it says otherwise. The order decides the
select's default and which host wins when discovery's guessed slug exists on
two.
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
which lets a trailing newline through) and refuses anything that could add a
path, a query, a fragment, a port or a user to the URLs built from it.
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
(one DNS label) cover the usual cases; a slug with several parts joins them
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
**Patterns.** The slug is group 1, or, when a pattern has more than one
group, the groups that matched joined by "/". Only group 1 is checked
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
paths are case-sensitive (Workday's site), since robots.txt paths are too. A
pattern without a group names the host with no slug. A host that
`discover.py` names in its own `_UNREADABLE` table is taken from its module
instead as soon as the module exists.
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
name; `_shared.only(host)` is the usual one. Every request goes through
`get`, which is `ats._get` bound to the board:
get(url, *, method="GET", payload=None, headers=None) -> bytes
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
default User-Agent and `Accept: application/json, text/xml`, and replace
either where they name it. `get` is https only on port 443, never to an IP
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
most three redirects each checked against `allowed(slug)`, cuts the body at
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
failure, and another exception (`ValueError` among them) for a refused host,
an oversized body or a broken redirect. A module lets these through; the
shared code logs, records and decides.
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
id, to decide completeness), the total the host states or None, and the
cursor of the next page or None after the last. The first call gets cursor
None. A board with a total is complete when as many distinct ids were read;
one without, when the read ended on an empty page. A 429 after the first page
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
apart. A paged module may also have `parse(body, slug)` for one page's body.
**Rows.** A `Row` is a dict with `source_id` (the host's own id for the
role, unique on the board), `title`, `location`, `url` (http or https,
checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see
`_shared.parse_dt`), `department`, `description` (plain text, see
`desk.sources.strip_html`), `remote` (True, False, a word or None) and
`salary_text`; and, where the host states pay, `salary_min`, `salary_max`
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`.
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
dict of the fields it found. The shared code calls it for roles not yet read
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
and `posted`, the keys whose columns survive a later pull that sends the
listing's empty value; any other key is left out with a debug log line.
`title`, `url`, `department` and `salary_text` cannot be filled from
`detail`: the listing rewrites them every pull. A role not read in detail
on a pull is sent with an empty description, so the one read earlier stays.
A 404 or a 410 counts as read; any other failure is logged and retried on a
later pull; a 429 ends the detail reads of that board for the pull.
A role read in detail on an earlier pull is sent without the
listing's place and remote, so what its own page said about them stays.
Completeness is decided by the listing alone.
**robots.txt.** With `robots_page(slug)`, the shared code reads
`https://<its host>/robots.txt` through `get` before the listing, at most
once a day per board (`ats:robots:<target>`), following a redirect to the
host's `www.` twin or back and no other. The board fails with "robots.txt
disallows it" when the rules for `*` or for the app's own token disallow the
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
final `$` in paths, case-sensitive paths, or without case for a host with
`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as
Workday does). A 4xx other than 429 means no
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
the board with "robots.txt could not be read" and is asked again next pull.
`robots_page` must be on a host `allowed(slug)` admits.
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
the slug is a board; `probe_ok(body)` narrows that when a host answers an
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
name is probed as, `(guess,)` without it. A host without `probe_url` is never
probed, only found by a link.
"""
from __future__ import annotations
import importlib
import pkgutil
import re
from collections.abc import Iterable
from types import ModuleType
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
DEFAULT_RANK = 100
#: What every host module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal."""
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
out.append("ROBOTS_ANY_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
patterns = getattr(module, "PATTERNS", ())
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
out.append("PATTERNS is not a tuple of strings")
else:
for pattern in patterns:
try:
re.compile(pattern)
except (TypeError, re.error) as exc:
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
out += [f"{attr} is not callable" for attr in FUNCTIONS
if hasattr(module, attr) and not callable(getattr(module, attr))]
if hasattr(module, "page"):
if not hasattr(module, "rows"):
out.append("page without rows")
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
out.append("neither feed_url and parse nor page and rows")
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
out.append("probe_ok without probe_url")
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
out.append("probe_slugs without probe_url")
return out
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
"""`NAME -> module` for every module of `package` (found on `path`) whose
name does not start with `_`, in order of `(RANK, NAME)`. Raises
ImportError naming the first module that breaks the contract."""
modules: list[ModuleType] = []
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{package}.{info.name}")
if found := problems(module, info.name):
raise ImportError(f"{package}.{info.name} breaks the host contract: "
+ "; ".join(found))
modules.append(module)
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
return {module.NAME: module for module in modules}
#: host -> its module, in order of `(RANK, NAME)`.
HOSTS: dict[str, ModuleType] = load(__path__, __name__)
+329
View File
@@ -0,0 +1,329 @@
"""The readers: one module per source of roles, a host's boards or a search.
A reader registers by existing: every module in this package whose name does
not start with `_` is loaded when the package is imported, checked against
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
`HOSTS` holds those of `scope="board"` (and every module without `READS`),
`PROVIDERS` those of `scope="search"`. A module that breaks the contract
fails the import with an ImportError naming the module and every problem,
rather than a pull that fails board by board. `plugins/ats.py` builds its
allow-list, feeds, pages, the Sources page's host select and its
description from `HOSTS`, and `app/jobs/discover.py` its patterns and
probes. The package sits beside `ats.py` rather than being `plugins/ats/`,
since `desk.sources.registry` loads `plugins/*.py` by path and the tests
patch names on the `ats` module. The types are in `._contract`.
A module is named after its source (`plugins/readers/workday.py`) and
imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It
holds:
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
LABEL = "Workday" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int; lower comes first in the select and the probes
KEEP_CASE = False # optional; True keeps a discovered slug's case
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000,
text="detail", complete="stated",
filters=Filters(country="ids"), places="many",
fields=Fields(posted="listed"),
robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)")
def slug_ok(slug: str) -> bool # strict, see below
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
def careers_url(slug: str) -> str # the page a person opens
and the functions its `READS` names:
def feed_url(slug: str, ask: Ask) -> str # listing="whole": one answer,
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged",
def rows(decoded: Any, slug: str) -> list[Row] # a page a call
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
and, optionally:
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
# read once when `feed_url` is over the cap
def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt)
def probe_url(slug: str) -> str # discovery: 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its
slug is the provider's own (a search reads no board), and it is not
discovered. A module without `READS` is read as a board with the older
signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by
`undeclared()`; the conformance test marks it until it declares.
**Declaration.** `READS` states what the reader does; the runner does it.
`problems()` checks it against the functions: `paged` needs `page` and
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key`
names a config entry, `robots` is not empty, and each function takes the
arguments above. The traits:
- `scope`: `board`, one company's whole list; `search`, many companies,
never a whole list, so nothing a search stops listing is closed by it.
- `listing`: `whole`, one request; `paged`, with `page_size` and `ceiling`,
the offset past which the host answers nothing whatever is asked. A board
above its ceiling is incomplete unless a filter brings it under.
- `text`: `listing`, the list has the full text; `excerpt`, a teaser and no
more; `detail`, one request a role; `none`. `teaser=True` with `detail`
says the list carries a teaser as well.
- `complete`: `stated`, the host gives a total; `end`, an empty page ends
the list; `whole`, the one answer is all; `never`.
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
`one` (one value a request), `many` (a list in one request), `ids` (host
ids looked up with `facets`).
- `places`: `one`, `many`, or `copies` when the host publishes one posting
per place with its own id.
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
posting date; `listed`, a date the host moves; `none`), and `remote`,
`country`, `seniority`, `employment` (`stated` or `none`).
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
and its environment fallbacks. The runner resolves it and passes it in
`Ask.params["key"]`.
- `robots`: the robots.txt decision, see below. Required.
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
far under the last count counts as failed.
- `title`: `provisional` when the listing may lack the title and `detail`
fills it.
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
wants from one series of requests. It sets only dimensions the reader
declared. With `one` it runs a series per value and unions the rows by
`source_id`. With `ids`, `countries` and `places` hold host ids, which the
runner maps from `facets(get, slug)`: `{"country": {"DE": "<host id>"},
"place": {...}}`, called at most once a day per board and kept in state.
`params` holds a search's saved params. `Ask()` is an unfiltered read. A
reader turns `Ask` into its host's query in `feed_url` or `page` and never
filters rows itself.
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
hold 10 to 80, which keeps the order they had before there were modules, so
a new host comes after them unless it says otherwise. The order decides the
select's default and which host wins when discovery's guessed slug exists on
two.
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
which lets a trailing newline through) and refuses anything that could add a
path, a query, a fragment, a port or a user to the URLs built from it.
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
(one DNS label) cover the usual cases; a slug with several parts joins them
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
**Patterns.** The slug is group 1, or, when a pattern has more than one
group, the groups that matched joined by "/". Only group 1 is checked
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
paths are case-sensitive (Workday's site). A pattern without a group names
the host with no slug. A host that `discover.py` names in its own
`_UNREADABLE` table is taken from its module instead as soon as the module
exists.
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
name; `_shared.only(host)` is the usual one. Every request goes through
`get`, which is `ats._get` bound to the board:
get(url, *, method="GET", payload=None, headers=None) -> bytes
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
default User-Agent and `Accept: application/json, text/xml`, and replace
either where they name it. `get` is https only on port 443, never to an IP
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
most three redirects each checked against `allowed(slug)`, cuts the body at
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
failure, and another exception (`ValueError` among them) for a refused host,
an oversized body or a broken redirect. A module lets these through; the
shared code logs, records and decides.
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
id, to decide completeness), the total the host states or None, and the
cursor of the next page or None after the last. The first call gets cursor
None. A board with a total is complete when as many distinct ids were read;
one without, when the read ended on an empty page. A 429 after the first page
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
apart. A paged module may also have `parse(body, slug)` for one page's body.
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
own id for the role, unique on the board), `title`, `url` (http or https,
checked with `_shared.is_web_url`) and `places`; `location` (the host's
one-line place, kept while storage reads it), `posted` (an aware datetime
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
see `desk.sources.strip_html`), `remote` (True, False, a word or None) and
`salary_text`; where the host states pay, `salary_min`, `salary_max`
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a
new field needs no change there. Undeclared modules do not state `places`
yet.
**Places.** A role has a list of places: `places` is `[Place(text=...,
remote=...)]`, one entry per place the host states, in the host's order, in
the host's words. A reader never multiplies a row per place, never drops a
place and never joins places into one `text`. `remote` is True for a remote
place, False for an office, None when unsaid. A host that publishes one
posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role
that keeps each copy's link.
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
over as `teaser`, plain text, whether or not the full text comes later. It
is shown and embedded until `description` arrives. A reader that sees one
keeps it.
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
dict of the fields it found. The shared code calls it for roles not yet read
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
and `posted`, the keys whose columns survive a later pull that sends the
listing's empty value; any other key is left out with a debug log line.
`title`, `url`, `department` and `salary_text` cannot be filled from
`detail`: the listing rewrites them every pull. A role not read in detail
on a pull is sent with an empty description, so the one read earlier stays.
A 404 or a 410 counts as read; any other failure is logged and retried on a
later pull; a 429 ends the detail reads of that board for the pull.
A role read in detail on an earlier pull is sent without the
listing's place and remote, so what its own page said about them stays.
Completeness is decided by the listing alone.
**robots.txt.** Decided when the reader is built, not checked at run time.
Whoever builds a reader reads the host's robots.txt for the paths the reader
requests and reports what it disallows, and the owner decides whether the
host is read anyway. `robots` records that decision with its date, e.g.
`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is
disallowed: owner, 2026-09-30"`. The runtime check in `ats.py`, which
`robots_page` and `ROBOTS_ANY_CASE` feed, predates this rule and goes with
the runner; a new reader defines neither. Until then it reads
`https://<host>/robots.txt` at most once a day per board and fails the board
when the page is disallowed for `*` or the app's token.
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
the slug is a board; `probe_ok(body)` narrows that when a host answers an
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
name is probed as, `(guess,)` without it. A host without `probe_url` is never
probed, only found by a link.
"""
from __future__ import annotations
import importlib
import pkgutil
import re
from collections.abc import Iterable
from types import ModuleType
from . import _contract
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
"Place", "Reads", "Row", "load", "problems", "scope", "undeclared"]
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
DEFAULT_RANK = 100
#: What every board module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: What a module of `scope="search"` defines.
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok",
"probe_slugs")
def scope(module: ModuleType) -> str:
"""`board` or `search`; `board` for a module without `READS`."""
reads = getattr(module, "READS", None)
return reads.scope if isinstance(reads, Reads) else "board"
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal. `READS` is checked when the
module declares one (`_contract.problems`)."""
required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
out.append("ROBOTS_ANY_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
patterns = getattr(module, "PATTERNS", ())
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
out.append("PATTERNS is not a tuple of strings")
else:
for pattern in patterns:
try:
re.compile(pattern)
except (TypeError, re.error) as exc:
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
out += [f"{attr} is not callable" for attr in FUNCTIONS
if hasattr(module, attr) and not callable(getattr(module, attr))]
if hasattr(module, "page"):
if not hasattr(module, "rows"):
out.append("page without rows")
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
out.append("neither feed_url and parse nor page and rows")
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
out.append("probe_ok without probe_url")
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
out.append("probe_slugs without probe_url")
return out + _contract.problems(module)
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
"""`NAME -> module` for every module of `package` (found on `path`) whose
name does not start with `_`, in order of `(RANK, NAME)`. Raises
ImportError naming the first module that breaks the contract."""
modules: list[ModuleType] = []
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{package}.{info.name}")
if found := problems(module, info.name):
raise ImportError(f"{package}.{info.name} breaks the reader contract: "
+ "; ".join(found))
modules.append(module)
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
return {module.NAME: module for module in modules}
def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]:
"""The names of the modules (of `READERS` by default) without `READS`,
read with the older signatures until they declare."""
return [name for name, module in (READERS if modules is None else modules).items()
if not hasattr(module, "READS")]
#: name -> module for every reader, in order of `(RANK, NAME)`.
READERS: dict[str, ModuleType] = load(__path__, __name__)
#: host -> its module: the readers of `scope="board"` and the undeclared.
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if scope(module) == "board"}
#: provider -> its module: the readers of `scope="search"`.
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if scope(module) == "search"}
+291
View File
@@ -0,0 +1,291 @@
"""The types of the reader contract, and the check of a module's declaration
against its functions. The prose is the package's docstring; this file is
what it names.
A reader declares `READS = Reads(...)`: what it reads (a board or a search),
how the listing arrives, where the text is, how completeness is known, what
it can filter at the source, how it states places and which fields it
states. The runner reads the declaration and does the rest. Values are
checked by pydantic where the module is declared, so a misspelt trait fails
the import; what depends on the module's functions, or on two traits at
once, is `problems()`, which the package's loader runs and turns into an
ImportError naming every problem.
Signatures of a module that declares `READS`, with `Get` and `Page` from
`._shared`:
def feed_url(slug: str, ask: Ask) -> str # listing="whole"
def parse(body: bytes, slug: str) -> list[Row] # listing="whole"
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged"
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
A module without `READS` keeps the signatures without `ask` until it
declares; `undeclared()` in the package lists those modules.
"""
from __future__ import annotations
import inspect
import re
from collections.abc import Callable
from types import ModuleType
from typing import Any, Literal, Required, TypedDict
from pydantic import ConfigDict
from desk.models import Model
#: A filter dimension's reach at the source: `none`, not filtered; `one`, one
#: value a request (the runner makes one series per value and unions them by
#: id); `many`, a list in one request; `ids`, values the host names by its own
#: ids, which `facets` looks up.
Filter = Literal["none", "one", "many", "ids"]
#: Whether the reader states a field: `stated` from the host, `none` never.
Stated = Literal["stated", "none"]
#: A config entry name: `sources.search.<name>` in `config.yaml`.
_CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
#: An environment variable name.
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
class _Frozen(Model):
"""A declaration: immutable, extra keys refused."""
model_config = ConfigDict(from_attributes=True, extra="forbid", frozen=True)
class Filters(_Frozen):
"""What the host filters at the source, per dimension. `country` takes
ISO 3166 alpha-2 codes, `place` place names, `remote` a bool, `query`
words. The runner never hands a reader a value on a dimension it
declared `none`."""
country: Filter = "none"
place: Filter = "none"
remote: Filter = "none"
query: Filter = "none"
def declared(self) -> dict[str, Filter]:
"""The dimensions filtered at the source, `name -> reach`."""
return {name: reach for name, reach in self.model_dump().items() if reach != "none"}
class Fields(_Frozen):
"""Which fields the rows state, and how far they can be trusted.
`pay`: `figures` (`salary_min`, `salary_max`, `currency`), `line` (only
`salary_text`), `none`. `posted`: `trusted` (the host's date is the
posting date), `listed` (a date, but one the host moves, e.g. on an
edit), `none`. The rest: `stated` when the host states it, `none` when
the app has to infer it.
"""
pay: Literal["figures", "line", "none"] = "none"
posted: Literal["trusted", "listed", "none"] = "none"
remote: Stated = "none"
country: Stated = "none"
seniority: Stated = "none"
employment: Stated = "none"
class Key(_Frozen):
"""An API key the reader needs: the config entry `sources.search.<name>`
with `fields`, each falling back to the environment variable at the same
position in `env`. The runner resolves it and hands the reader the
values in `Ask.params["key"]`; a reader never reads a key itself."""
name: str
fields: tuple[str, ...]
env: tuple[str, ...] = ()
class Reads(_Frozen):
"""What a reader reads and how. One per module, as `READS`.
`scope`: `board` (one company's whole list) or `search` (many companies,
never a whole list). `listing`: `whole` (one answer) or `paged`.
`page_size` and `ceiling` (paged only): roles a page, and the offset
where the host stops answering whatever is asked (Workday 2,000).
`text`: `listing` (full text in the list), `excerpt` (a teaser in the
list, nothing more), `detail` (one request a role), `none`. `teaser`:
with `text="detail"`, the list also carries a teaser, handed over as
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
`whole` (the one answer), `never` (a search). `places`: `one` (the host
states one), `many` (a list per role), `copies` (the host lists one
posting per place, each with its own id). `robots`: the robots.txt
decision, with the date, taken when the reader was built. `empty`:
`suspect` when an unknown slug answers an empty list rather than an
error. `title`: `provisional` when the listing may not state the title
(a sitemap), which detail then fills.
"""
scope: Literal["board", "search"]
listing: Literal["whole", "paged"]
page_size: int | None = None
ceiling: int | None = None
text: Literal["listing", "excerpt", "detail", "none"]
teaser: bool = False
complete: Literal["stated", "end", "whole", "never"]
filters: Filters = Filters()
places: Literal["one", "many", "copies"]
fields: Fields = Fields()
robots: str
empty: Literal["trusted", "suspect"] = "trusted"
key: Key | None = None
title: Literal["final", "provisional"] = "final"
class Ask(_Frozen):
"""What the runner wants from one series of requests. Only the
dimensions the reader declared are set; for a `one` filter each series
carries one value. With `ids`, `countries` and `places` hold the host's
ids from `facets`, not codes or names. `params` are a search's saved
params (and a resolved key under `"key"`). An empty `Ask()` is an
unfiltered read."""
countries: tuple[str, ...] = ()
places: tuple[str, ...] = ()
remote: bool | None = None
query: str = ""
params: dict[str, Any] = {}
class Place(TypedDict):
"""One place of a role, as the host states it: `text` the host's own
words ("Berlin, Germany"), `remote` True for a remote place ("Remote
(DE)"), False for an office, None when the host does not say. Resolving
`text` to a city, region or country is the app's job, not the reader's."""
text: str
remote: bool | None
class Row(TypedDict, total=False):
"""One role as a reader hands it over.
`source_id` is the host's id, unique within the board or the search;
`url` a web URL (`_shared.is_web_url`); `places` every place the host
states, never multiplied into rows, never cut to one; `teaser` the
short text the list carries. `location` is the host's own one-line
place, kept while storage reads it. `posted` is an aware datetime or
None; `description` plain text; `remote` True, False, a word or None;
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
`employment` only where the host states them, as `Fields` declares. Any
other stated key is kept in `raw` by the runner.
"""
source_id: Required[str]
title: Required[str]
url: Required[str]
places: Required[list[Place]]
teaser: str
location: str
posted: Any
department: str
description: str
remote: bool | str | None
salary_text: str
salary_min: int | None
salary_max: int | None
currency: str | None
country: str | None
seniority: str | None
employment: str | None
#: What a declared module's functions take, positionally.
SIGNATURES: dict[str, tuple[str, ...]] = {
"feed_url": ("slug", "ask"),
"parse": ("body", "slug"),
"page": ("get", "slug", "cursor", "ask"),
"rows": ("decoded", "slug"),
"detail": ("get", "slug", "row"),
"facets": ("get", "slug"),
}
def _takes(function: Callable[..., Any], count: int) -> bool:
"""Whether `function` accepts `count` positional arguments."""
try:
signature = inspect.signature(function)
except (TypeError, ValueError):
return True
try:
signature.bind(*([None] * count))
except TypeError:
return False
return True
def problems(module: ModuleType) -> list[str]:
"""What `module`'s `READS` gets wrong, on its own or against the
module's functions; empty for a module that declares none."""
if not hasattr(module, "READS"):
return []
reads = module.READS
if not isinstance(reads, Reads):
return ["READS is not a Reads"]
has = lambda name: callable(getattr(module, name, None)) # noqa: E731
out: list[str] = []
if reads.listing == "paged" and not (has("page") and has("rows")):
out.append('listing="paged" without page and rows')
if reads.listing == "whole" and not (has("feed_url") and has("parse")):
out.append('listing="whole" without feed_url and parse')
if reads.listing == "whole" and has("page"):
out.append('listing="whole" with page')
if reads.listing == "whole" and (reads.page_size is not None or reads.ceiling is not None):
out.append('page_size and ceiling are for listing="paged"')
if reads.page_size is not None and reads.page_size < 1:
out.append("page_size is not positive")
if reads.ceiling is not None and reads.ceiling < (reads.page_size or 1):
out.append("ceiling is below page_size")
if has("fallback_url") and reads.listing != "whole":
out.append('fallback_url without listing="whole"')
if reads.text == "detail" and not has("detail"):
out.append('text="detail" without detail')
if has("detail") and reads.text != "detail":
out.append(f'detail with text="{reads.text}"')
if reads.title == "provisional" and not has("detail"):
out.append('title="provisional" without detail to fill it')
if reads.teaser and reads.text != "detail":
out.append('teaser is for text="detail"; the list of "excerpt" is the teaser')
if (reads.complete == "whole") != (reads.listing == "whole"):
out.append('complete="whole" goes with listing="whole" and only with it')
if reads.scope == "search" and reads.complete not in ("never", "stated"):
out.append('scope="search" needs complete="never" or "stated"')
if reads.scope == "board" and reads.complete == "never":
out.append('scope="board" cannot be complete="never"')
ids = [name for name, reach in reads.filters.declared().items() if reach == "ids"]
if ids and not has("facets"):
out.append(f"an ids filter ({', '.join(ids)}) without facets")
if has("facets") and not ids:
out.append("facets without an ids filter")
if reads.filters.query == "ids":
out.append('query cannot be "ids"')
if key := reads.key:
if not _CONFIG_NAME.fullmatch(key.name):
out.append(f"key names no config entry ({key.name!r})")
if not key.fields:
out.append("key has no fields")
if key.env and len(key.env) != len(key.fields):
out.append("key.env does not match key.fields")
if any(not _ENV_NAME.fullmatch(name) for name in key.env):
out.append("key.env has a name that is not an environment variable")
if not reads.robots.strip():
out.append("robots is empty: record the robots.txt decision and its date")
for name, params in SIGNATURES.items():
if has(name) and not _takes(getattr(module, name), len(params)):
out.append(f"{name} does not take ({', '.join(params)})")
return out
@@ -3,7 +3,7 @@ handed, and the date, URL and salary helpers more than one host needs.
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
import it without either importing the other. The leading underscore keeps import it without either importing the other. The leading underscore keeps
`plugins.atshosts` from listing it as a host. `plugins.readers` from listing it as a host.
""" """
from __future__ import annotations from __future__ import annotations
@@ -16,7 +16,9 @@ from typing import Any, Protocol
import httpx import httpx
#: What every parser turns a feed into: one dict per row, with the keys #: What an undeclared module's parser turns a feed into; a module that
#: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`.
#: One dict per row, with the keys
#: `source_id`, `title`, `location`, `url`, `posted`, `department`, #: `source_id`, `title`, `location`, `url`, `posted`, `department`,
#: `description`, `remote` and `salary_text`, and, where a host states pay, #: `description`, `remote` and `salary_text`, and, where a host states pay,
#: `salary_min`, `salary_max` and `currency`. #: `salary_min`, `salary_max` and `currency`.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+1
View File
@@ -96,6 +96,7 @@ ignore = [
"app/models.py" = ["RUF012"] "app/models.py" = ["RUF012"]
"desk/models.py" = ["RUF012"] "desk/models.py" = ["RUF012"]
"desk/sources/spec.py" = ["RUF012"] "desk/sources/spec.py" = ["RUF012"]
"plugins/readers/_contract.py" = ["RUF012"]
# The boilerplate's own test, adapted here but kept in its style. # The boilerplate's own test, adapted here but kept in its style.
"tests/test_coupling.py" = ["SIM102"] "tests/test_coupling.py" = ["SIM102"]
+1 -1
View File
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import ats # noqa: E402 from plugins import ats # noqa: E402
from plugins.atshosts import smartrecruiters # noqa: E402 from plugins.readers import smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
+4 -4
View File
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import bamboohr # noqa: E402 from plugins.readers import bamboohr # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -161,7 +161,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
assert allow("configit.bamboohr.com") and allow("CONFIGIT.bamboohr.com") assert allow("configit.bamboohr.com") and allow("CONFIGIT.bamboohr.com")
assert not allow("bamboohr.com") and not allow("other.bamboohr.com") assert not allow("bamboohr.com") and not allow("other.bamboohr.com")
assert not allow("configit.bamboohr.com.evil.example") and not allow("localhost") assert not allow("configit.bamboohr.com.evil.example") and not allow("localhost")
assert atshosts.HOSTS["bamboohr"] is bamboohr and ats.HOSTS["bamboohr"] == "BambooHR" assert readers.HOSTS["bamboohr"] is bamboohr and ats.HOSTS["bamboohr"] == "BambooHR"
@pytest.mark.parametrize("slug", [ @pytest.mark.parametrize("slug", [
@@ -182,4 +182,4 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None:
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>' '<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
'<a href="https://resources.bamboohr.com/y">z</a>') '<a href="https://resources.bamboohr.com/y">z</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")] assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS} assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
+3 -3
View File
@@ -16,8 +16,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import deel # noqa: E402 from plugins.readers import deel # noqa: E402
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes() PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
URL = "https://jobs.deel.com/klarna" URL = "https://jobs.deel.com/klarna"
@@ -126,7 +126,7 @@ def test_a_page_that_cannot_be_read_fails_the_board(web: dict[str, Any]) -> None
# ------------------------------------------------------- slugs and hosts # ------------------------------------------------------- slugs and hosts
def test_the_module_is_registered_as_a_feed_host() -> None: def test_the_module_is_registered_as_a_feed_host() -> None:
assert atshosts.HOSTS["deel"] is deel and ats.HOSTS["deel"] == "Deel" assert readers.HOSTS["deel"] is deel and ats.HOSTS["deel"] == "Deel"
assert deel.NAME == "deel" and deel.KEEP_CASE assert deel.NAME == "deel" and deel.KEEP_CASE
assert not hasattr(deel, "page") and not hasattr(deel, "detail") assert not hasattr(deel, "page") and not hasattr(deel, "detail")
+4 -4
View File
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import eightfold # noqa: E402 from plugins.readers import eightfold # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "jobs.ericsson.com/ericsson.com" SLUG = "jobs.ericsson.com/ericsson.com"
@@ -169,7 +169,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
assert allow("jobs.ericsson.com") and allow("JOBS.ERICSSON.COM") assert allow("jobs.ericsson.com") and allow("JOBS.ERICSSON.COM")
assert not allow("ericsson.com") and not allow("evil.example") assert not allow("ericsson.com") and not allow("evil.example")
assert not allow("jobs.ericsson.com.evil.example") and not allow("localhost") assert not allow("jobs.ericsson.com.evil.example") and not allow("localhost")
assert atshosts.HOSTS["eightfold"] is eightfold and ats.HOSTS["eightfold"] == "Eightfold" assert readers.HOSTS["eightfold"] is eightfold and ats.HOSTS["eightfold"] == "Eightfold"
@pytest.mark.parametrize("slug", [ @pytest.mark.parametrize("slug", [
@@ -192,4 +192,4 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None
found = [(h, s) for h, s, _ in discover.boards_in(page)] found = [(h, s) for h, s, _ in discover.boards_in(page)]
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
assert ("eightfold", "") in found assert ("eightfold", "") in found
assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS} assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
+4 -4
View File
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import gem # noqa: E402 from plugins.readers import gem # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
API = "https://jobs.gem.com/api/public/graphql" API = "https://jobs.gem.com/api/public/graphql"
@@ -154,7 +154,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
assert allow("jobs.gem.com") and allow("JOBS.gem.com") assert allow("jobs.gem.com") and allow("JOBS.gem.com")
assert not allow("gem.com") and not allow("productboard.gem.com") assert not allow("gem.com") and not allow("productboard.gem.com")
assert not allow("jobs.gem.com.evil.example") and not allow("localhost") assert not allow("jobs.gem.com.evil.example") and not allow("localhost")
assert atshosts.HOSTS["gem"] is gem and ats.HOSTS["gem"] == "Gem" assert readers.HOSTS["gem"] is gem and ats.HOSTS["gem"] == "Gem"
@pytest.mark.parametrize("slug", [ @pytest.mark.parametrize("slug", [
@@ -177,4 +177,4 @@ def test_discovery_finds_the_board_keeping_its_case() -> None:
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>') '<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [ assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
("gem", "Productboard"), ("gem", "sennder-gmbh")] ("gem", "Productboard"), ("gem", "sennder-gmbh")]
assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS} assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
+3 -3
View File
@@ -16,8 +16,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import hibob # noqa: E402 from plugins.readers import hibob # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hibob.json").read_bytes() BOARD = (FIXTURES / "hibob.json").read_bytes()
@@ -171,7 +171,7 @@ def test_the_robots_txt_that_is_the_apps_own_page_allows(web: dict[str, Any]) ->
# ------------------------------------------------------- slugs and hosts # ------------------------------------------------------- slugs and hosts
def test_the_module_is_registered_as_a_paged_host() -> None: def test_the_module_is_registered_as_a_paged_host() -> None:
assert atshosts.HOSTS["hibob"] is hibob and ats.HOSTS["hibob"] == "HiBob" assert readers.HOSTS["hibob"] is hibob and ats.HOSTS["hibob"] == "HiBob"
assert hibob.NAME == "hibob" and not getattr(hibob, "KEEP_CASE", False) assert hibob.NAME == "hibob" and not getattr(hibob, "KEEP_CASE", False)
assert hasattr(hibob, "page") and not hasattr(hibob, "detail") assert hasattr(hibob, "page") and not hasattr(hibob, "detail")
+3 -3
View File
@@ -15,8 +15,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import hrmanager # noqa: E402 from plugins.readers import hrmanager # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hrmanager.xml").read_bytes() BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
@@ -155,7 +155,7 @@ def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide(
# ------------------------------------------------------- slugs and hosts # ------------------------------------------------------- slugs and hosts
def test_the_module_is_registered_as_a_paged_host() -> None: def test_the_module_is_registered_as_a_paged_host() -> None:
assert atshosts.HOSTS["hrmanager"] is hrmanager and ats.HOSTS["hrmanager"] == "HR Manager" assert readers.HOSTS["hrmanager"] is hrmanager and ats.HOSTS["hrmanager"] == "HR Manager"
assert hrmanager.NAME == "hrmanager" and not getattr(hrmanager, "KEEP_CASE", False) assert hrmanager.NAME == "hrmanager" and not getattr(hrmanager, "KEEP_CASE", False)
assert hasattr(hrmanager, "page") and not hasattr(hrmanager, "detail") assert hasattr(hrmanager, "page") and not hasattr(hrmanager, "detail")
+3 -3
View File
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import jibe # noqa: E402 from plugins.readers import jibe # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SITE = "jobs.booking.com" SITE = "jobs.booking.com"
@@ -152,7 +152,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM") assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM")
assert not allow("booking.com") and not allow("evil.example") assert not allow("booking.com") and not allow("evil.example")
assert not allow("jobs.booking.com.evil.example") and not allow("localhost") assert not allow("jobs.booking.com.evil.example") and not allow("localhost")
assert atshosts.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe" assert readers.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe"
@pytest.mark.parametrize("slug", [ @pytest.mark.parametrize("slug", [
+3 -3
View File
@@ -16,8 +16,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import jobylon # noqa: E402 from plugins.readers import jobylon # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "jobylon.html").read_bytes() BOARD = (FIXTURES / "jobylon.html").read_bytes()
@@ -158,7 +158,7 @@ def test_an_empty_board_is_complete(web: dict[str, Any]) -> None:
# ------------------------------------------------------- slugs and hosts # ------------------------------------------------------- slugs and hosts
def test_the_module_is_registered_as_a_feed_host_with_detail() -> None: def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
assert atshosts.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon" assert readers.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon"
assert jobylon.NAME == "jobylon" and not getattr(jobylon, "KEEP_CASE", False) assert jobylon.NAME == "jobylon" and not getattr(jobylon, "KEEP_CASE", False)
assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page") assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page")
+2 -2
View File
@@ -20,8 +20,8 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402 from plugins import ats # noqa: E402
from plugins.atshosts import oracle # noqa: E402 from plugins.readers import oracle # noqa: E402
from plugins.atshosts._shared import is_web_url # noqa: E402 from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
HOST = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com" HOST = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com"
+2 -2
View File
@@ -19,8 +19,8 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402 from plugins import ats # noqa: E402
from plugins.atshosts import phenom # noqa: E402 from plugins.readers import phenom # noqa: E402
from plugins.atshosts._shared import is_web_url # noqa: E402 from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "careers.king.com/us/en" SLUG = "careers.king.com/us/en"
+4 -4
View File
@@ -14,8 +14,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import pinpoint # noqa: E402 from plugins.readers import pinpoint # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -122,7 +122,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com") assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com")
assert not allow("pinpointhq.com") and not allow("other.pinpointhq.com") assert not allow("pinpointhq.com") and not allow("other.pinpointhq.com")
assert not allow("xeneta.pinpointhq.com.evil.example") and not allow("localhost") assert not allow("xeneta.pinpointhq.com.evil.example") and not allow("localhost")
assert atshosts.HOSTS["pinpoint"] is pinpoint and ats.HOSTS["pinpoint"] == "Pinpoint" assert readers.HOSTS["pinpoint"] is pinpoint and ats.HOSTS["pinpoint"] == "Pinpoint"
@pytest.mark.parametrize("slug", [ @pytest.mark.parametrize("slug", [
@@ -146,7 +146,7 @@ def test_discovery_finds_the_subdomain() -> None:
'<script src="https://www.pinpointhq.com/embed.js"></script>') '<script src="https://www.pinpointhq.com/embed.js"></script>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")] assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
assert discover.boards_in("https://pinpointhq.com/") == [] assert discover.boards_in("https://pinpointhq.com/") == []
assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS} assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_a_pinpoint_board_is_not_probed() -> None: def test_a_pinpoint_board_is_not_probed() -> None:
+3 -3
View File
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import rippling # noqa: E402 from plugins.readers import rippling # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "rippling.json").read_bytes() BOARD = (FIXTURES / "rippling.json").read_bytes()
@@ -168,7 +168,7 @@ def test_robots_are_read_from_the_site_and_disallow_only_internal(
# ------------------------------------------------------- slugs and hosts # ------------------------------------------------------- slugs and hosts
def test_the_module_is_registered_as_a_feed_host_with_detail() -> None: def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
assert atshosts.HOSTS["rippling"] is rippling and ats.HOSTS["rippling"] == "Rippling" assert readers.HOSTS["rippling"] is rippling and ats.HOSTS["rippling"] == "Rippling"
assert rippling.NAME == "rippling" and rippling.KEEP_CASE assert rippling.NAME == "rippling" and rippling.KEEP_CASE
assert hasattr(rippling, "detail") and not hasattr(rippling, "page") assert hasattr(rippling, "detail") and not hasattr(rippling, "page")
+4 -4
View File
@@ -1,4 +1,4 @@
"""The SuccessFactors host: `plugins/atshosts/successfactors.py` against saved """The SuccessFactors host: `plugins/readers/successfactors.py` against saved
career-site pages. No network. The fixtures are trimmed pages of Vestas and career-site pages. No network. The fixtures are trimmed pages of Vestas and
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
with a path prefix), and the job pages of Vestas (`addressCountry`), with a path prefix), and the job pages of Vestas (`addressCountry`),
@@ -19,8 +19,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import successfactors # noqa: E402 from plugins.readers import successfactors # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -439,7 +439,7 @@ def test_careers_url_and_robots_page_are_the_search_page() -> None:
def test_the_module_is_case_sensitive_about_the_prefix() -> None: def test_the_module_is_case_sensitive_about_the_prefix() -> None:
assert successfactors.KEEP_CASE is True assert successfactors.KEEP_CASE is True
assert atshosts.HOSTS["successfactors"] is successfactors assert readers.HOSTS["successfactors"] is successfactors
assert successfactors._parts("jobs.teradyne.com/MiR") == ("jobs.teradyne.com", "/MiR") assert successfactors._parts("jobs.teradyne.com/MiR") == ("jobs.teradyne.com", "/MiR")
+1 -1
View File
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats # noqa: E402 from plugins import ats # noqa: E402
from plugins.atshosts import workday # noqa: E402 from plugins.readers import workday # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "zendesk.wd1/zendesk" SLUG = "zendesk.wd1/zendesk"
@@ -1,4 +1,4 @@
"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for """The host modules of `plugins/readers/` and what `plugins/ats.py` does for
every host: the registry and its contract check, the fetch a host is handed, every host: the registry and its contract check, the fetch a host is handed,
the per-role detail reads, robots.txt, and Lever's EU instance. No network — the per-role detail reads, robots.txt, and Lever's EU instance. No network —
a fake host stands in where a behaviour belongs to no real host yet.""" a fake host stands in where a behaviour belongs to no real host yet."""
@@ -22,8 +22,8 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path: if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import ats, atshosts # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.atshosts import lever # noqa: E402 from plugins.readers import lever # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -31,14 +31,15 @@ FIXTURES = ROOT / "tests" / "fixtures" / "ats"
# --------------------------------------------------------------- registry # --------------------------------------------------------------- registry
def test_the_registry_finds_every_module() -> None: def test_the_registry_finds_every_module() -> None:
names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py") names = sorted(p.stem for p in (ROOT / "plugins" / "readers").glob("*.py")
if not p.stem.startswith("_")) if not p.stem.startswith("_"))
assert sorted(atshosts.HOSTS) == names assert sorted(readers.READERS) == names
assert set(readers.HOSTS) | set(readers.PROVIDERS) == set(names)
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters", assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio", "teamtailor"} "recruitee", "personio", "teamtailor"}
assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items()) assert all(not readers.problems(module, name) for name, module in readers.READERS.items())
# The Sources page's select and the plugin's description follow the modules. # The Sources page's select and the plugin's description follow the modules.
assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS assert {name: module.LABEL for name, module in readers.HOSTS.items()} == ats.HOSTS
assert "Teamtailor" in ats.ATSSource.description assert "Teamtailor" in ats.ATSSource.description
@@ -86,7 +87,7 @@ def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
path = _package(tmp_path, monkeypatch, "hostsok", { path = _package(tmp_path, monkeypatch, "hostsok", {
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"), "zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
"_helpers": "X = 1\n"}) "_helpers": "X = 1\n"})
assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"] assert list(readers.load(path, "hostsok")) == ["alpha", "zeta"]
def test_the_registry_orders_by_rank_then_name( def test_the_registry_orders_by_rank_then_name(
@@ -95,7 +96,7 @@ def test_the_registry_orders_by_rank_then_name(
"alpha": _GOOD.format(name="alpha"), # the default, 100 "alpha": _GOOD.format(name="alpha"), # the default, 100
"beta": _GOOD.format(name="beta") + "RANK = 5\n", "beta": _GOOD.format(name="beta") + "RANK = 5\n",
"gamma": _GOOD.format(name="gamma") + "RANK = 100\n"}) "gamma": _GOOD.format(name="gamma") + "RANK = 100\n"})
assert list(atshosts.load(path, "hostsranked")) == ["beta", "alpha", "gamma"] assert list(readers.load(path, "hostsranked")) == ["beta", "alpha", "gamma"]
def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None: def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
@@ -107,8 +108,8 @@ def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
assert ats.ATSSource.description.startswith( assert ats.ATSSource.description.startswith(
"Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, " "Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, "
"Recruitee, Personio, Teamtailor") "Recruitee, Personio, Teamtailor")
assert all(module.RANK < atshosts.DEFAULT_RANK assert all(module.RANK < readers.DEFAULT_RANK
for name, module in atshosts.HOSTS.items() if name in first) for name, module in readers.HOSTS.items() if name in first)
def test_the_registry_refuses_a_module_missing_a_required_name( def test_the_registry_refuses_a_module_missing_a_required_name(
@@ -116,7 +117,7 @@ def test_the_registry_refuses_a_module_missing_a_required_name(
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url") broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken}) path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"): with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
atshosts.load(path, "hostsbad") readers.load(path, "hostsbad")
def test_the_contract_check_names_each_problem() -> None: def test_the_contract_check_names_each_problem() -> None:
@@ -126,7 +127,7 @@ def test_the_contract_check_names_each_problem() -> None:
module.page = lambda get, slug, cursor: None module.page = lambda get, slug, cursor: None
module.probe_ok = lambda body: True module.probe_ok = lambda body: True
module.RANK, module.KEEP_CASE = "1", "yes" module.RANK, module.KEEP_CASE = "1", "yes"
found = atshosts.problems(module, "odd") found = readers.problems(module, "odd")
assert "NAME is 'other', not the module's name 'odd'" in found assert "NAME is 'other', not the module's name 'odd'" in found
assert "LABEL is not a non-empty string" in found assert "LABEL is not a non-empty string" in found
assert any(p.startswith("PATTERNS has '('") for p in found) assert any(p.startswith("PATTERNS has '('") for p in found)
@@ -294,7 +295,7 @@ def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType, def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
state: dict[str, str] | None = None) -> tuple[list[Any], Any]: state: dict[str, str] | None = None) -> tuple[list[Any], Any]:
monkeypatch.setitem(atshosts.HOSTS, "fake", module) monkeypatch.setitem(readers.HOSTS, "fake", module)
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
if state is not None: if state is not None:
source.state = state source.state = state
@@ -411,7 +412,7 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
base = "https://jobs.fake.example/acme" base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'}) web["answers"].update({f"{base}/list": b'["2"]'})
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: ( monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False)) readers.HOSTS["fake"].parse(b'["2"]', "acme"), False))
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state) _items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
@@ -422,7 +423,7 @@ def test_fetch_board_reads_the_listing_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme" base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"}) web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
rows, complete = ats.fetch_board("fake:acme") rows, complete = ats.fetch_board("fake:acme")
assert ([r["description"] for r in rows], complete) == (["teaser"], True) assert ([r["description"] for r in rows], complete) == (["teaser"], True)
assert web["asked"] == [f"{base}/list"] assert web["asked"] == [f"{base}/list"]
@@ -534,7 +535,7 @@ def _paged_host() -> types.ModuleType:
module.page = page module.page = page
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug) module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
assert not atshosts.problems(module, "fake") assert not readers.problems(module, "fake")
return module return module
@@ -574,7 +575,7 @@ def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
stored, is read again on the next pull.""" stored, is read again on the next pull."""
base = "https://jobs.fake.example/acme" base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"}) web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
roles: Any = source.fetch() # a generator, which `close` stops roles: Any = source.fetch() # a generator, which `close` stops
assert next(roles).body == "one" assert next(roles).body == "one"
@@ -728,7 +729,7 @@ def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
def test_a_robots_txt_redirect_to_the_www_twin_is_followed( def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Disallowed): pytest.raises(ats._Disallowed):
ats._check_robots(client, "fake:acme", {}) ats._check_robots(client, "fake:acme", {})
@@ -740,13 +741,13 @@ def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
]) ])
def test_a_robots_txt_redirect_anywhere_else_is_unreachable( def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
monkeypatch: pytest.MonkeyPatch, location: str) -> None: monkeypatch: pytest.MonkeyPatch, location: str) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable): with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
ats._check_robots(client, "fake:acme", {}) ats._check_robots(client, "fake:acme", {})
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None: def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Refused): pytest.raises(ats._Refused):
ats._get(client, ROBOTS, "fake", "acme") ats._get(client, ROBOTS, "fake", "acme")
@@ -756,7 +757,7 @@ def test_fetch_board_honours_robots_txt(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
"https://jobs.fake.example/acme/list": b'["1"]'}) "https://jobs.fake.example/acme/list": b'["1"]'})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
with pytest.raises(ats._Disallowed) as caught: with pytest.raises(ats._Disallowed) as caught:
ats.fetch_board("fake:acme") ats.fetch_board("fake:acme")
assert ats.failure_reason(caught.value) == "robots.txt disallows it" assert ats.failure_reason(caught.value) == "robots.txt disallows it"
+387
View File
@@ -0,0 +1,387 @@
"""The reader contract, held for every module of `plugins/readers/`.
Per module: it declares `READS`, the declaration matches its functions, and
its rows keep what the contract says they keep. No network: fixtures stand
in for the host.
`UNDECLARED` lists the modules that do not declare `READS` yet; their tests
are marked xfail (not strict), and the planner removes a name once the
module declares. A module not in the list must pass.
Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/`
for searches, named after the module:
- `<name>-places.<ext>`: one listing answer with at least one role at two
or more places. For `listing="whole"` it is the body `parse` gets; for
`listing="paged"` it is what every request of the first `page` call
returns, and `rows` reads that page. Required by the test of a module
that declares `places="many"`, which is skipped until the fixture exists.
- `<name>-places-detail.<ext>`: optional, for a host whose places come from
the role's own page: what `detail`'s requests return; its `places` replace
the row's.
- `<name>-teaser.<ext>`: one listing answer with a teaser, read the same
way. Required by the test of a module with `text="excerpt"` or
`teaser=True`, skipped until it exists.
- `<name>.slug`: optional, the slug the fixtures are read with, one line;
`acme` without it.
"""
from __future__ import annotations
import sys
import types
from collections.abc import Callable
from pathlib import Path
from typing import Any
import pydantic
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402
from plugins.readers import _contract # noqa: E402
FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" / "search")
#: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and
#: C (the search providers, which arrive declared) land.
UNDECLARED = frozenset({
"ashby", "bamboohr", "deel", "eightfold", "gem", "greenhouse", "hibob", "hrmanager",
"jibe", "jobylon", "lever",
"oracle", "personio", "phenom", "pinpoint", "recruitee", "rippling", "smartrecruiters",
"successfactors", "teamtailor", "workable", "workday",
})
def _params() -> list[Any]:
return [pytest.param(name, marks=pytest.mark.xfail(reason="no READS yet", strict=False))
if name in UNDECLARED else name for name in readers.READERS]
def _declared(name: str) -> tuple[types.ModuleType, Reads]:
module = readers.READERS[name]
reads = getattr(module, "READS", None)
assert isinstance(reads, Reads), f"{name} declares no READS"
return module, reads
def _fixture(name: str, kind: str) -> Path | None:
for folder in FIXTURE_DIRS:
found = sorted(folder.glob(f"{name}-{kind}.*"))
if found:
return found[0]
return None
def _slug(name: str) -> str:
for folder in FIXTURE_DIRS:
if (path := folder / f"{name}.slug").exists():
return path.read_text(encoding="utf-8").strip()
return "acme"
def _answer(body: bytes) -> Callable[..., bytes]:
return lambda url, **kwargs: body
def read_fixture(module: types.ModuleType, body: bytes, slug: str,
detail_body: bytes | None = None) -> list[dict[str, Any]]:
"""The rows `module` makes of one listing answer, as the runner reads
them: `parse` for a whole listing, the first `page` and its `rows` for a
paged one, and, given `detail_body`, each row's `places` from `detail`."""
reads: Reads = module.READS
if reads.listing == "whole":
rows = list(module.parse(body, slug))
else:
decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, Ask())
rows = list(module.rows(decoded, slug))
if detail_body is not None:
for row in rows:
found = module.detail(_answer(detail_body), slug, row)
if "places" in found:
row["places"] = found["places"]
return rows
def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
"""What rows get wrong about `places` under the declared trait."""
out = []
for row in rows:
found = row.get("places")
rid = row.get("source_id")
if not isinstance(found, list):
out.append(f"{rid}: places is not a list")
continue
for place in found:
if not (isinstance(place, dict) and set(place) == {"text", "remote"}
and isinstance(place["text"], str) and place["text"].strip()
and place["remote"] in (True, False, None)):
out.append(f"{rid}: {place!r} is not a Place")
if places in ("one", "copies") and len(found) > 1:
out.append(f'{rid}: {len(found)} places under places="{places}"')
if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows):
out.append('no row has two places: a reader with places="many" keeps every place')
return out
# ------------------------------------------------------------ every module
def test_the_undeclared_list_names_only_readers() -> None:
assert set(readers.READERS) >= UNDECLARED
def test_undeclared_lists_the_modules_without_reads() -> None:
assert set(readers.undeclared()) == {name for name, module in readers.READERS.items()
if not hasattr(module, "READS")}
@pytest.mark.parametrize("name", _params())
def test_a_reader_declares_reads(name: str) -> None:
_declared(name)
@pytest.mark.parametrize("name", _params())
def test_a_declaration_matches_the_functions(name: str) -> None:
module, _reads = _declared(name)
assert readers.problems(module, name) == []
expected = "search" if module.READS.scope == "search" else "board"
assert name in (readers.PROVIDERS if expected == "search" else readers.HOSTS)
@pytest.mark.parametrize("name", _params())
def test_a_reader_keeps_every_place(name: str) -> None:
module, reads = _declared(name)
fixture = _fixture(name, "places")
if fixture is None:
if reads.places == "many":
pytest.skip(f"no fixture {name}-places.* yet")
pytest.skip(f'places="{reads.places}" and no fixture')
detail = _fixture(name, "places-detail")
rows = read_fixture(module, fixture.read_bytes(), _slug(name),
detail.read_bytes() if detail else None)
assert rows, f"{fixture.name} gave no rows"
assert place_problems(rows, reads.places) == []
@pytest.mark.parametrize("name", _params())
def test_a_reader_keeps_the_teaser(name: str) -> None:
module, reads = _declared(name)
if not (reads.text == "excerpt" or reads.teaser):
pytest.skip("the list carries no teaser")
fixture = _fixture(name, "teaser")
if fixture is None:
pytest.skip(f"no fixture {name}-teaser.* yet")
rows = read_fixture(module, fixture.read_bytes(), _slug(name))
assert any(isinstance(row.get("teaser"), str) and row["teaser"].strip() for row in rows)
# ------------------------------------------------------------ the checks
def _reader(**functions: Any) -> Any:
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", (r"fake\.example/([a-z]+)",)
module.slug_ok = lambda slug: True
module.allowed = lambda slug: lambda host: host == "fake.example"
module.careers_url = lambda slug: "https://fake.example/" + slug
for name, function in functions.items():
setattr(module, name, function)
return module
def _whole(**functions: Any) -> Any:
return _reader(**{"feed_url": lambda slug, ask: "https://fake.example/" + slug,
"parse": lambda body, slug: []} | functions)
def _paged(**functions: Any) -> Any:
return _reader(**{"page": lambda get, slug, cursor, ask: (None, [], None, None),
"rows": lambda decoded, slug: []} | functions)
_ROBOTS = "read: no robots.txt rule on /api/ (checked 2026-09-30)"
def _reads(**traits: Any) -> Reads:
base: dict[str, Any] = {"scope": "board", "listing": "whole", "text": "listing",
"complete": "whole", "places": "one", "robots": _ROBOTS}
return Reads(**(base | traits))
def test_a_good_declaration_has_no_problems() -> None:
module = _whole()
module.READS = _reads()
assert readers.problems(module, "fake") == []
module = _paged(detail=lambda get, slug, row: {},
facets=lambda get, slug: {"country": {"DE": "1"}})
module.READS = _reads(listing="paged", page_size=20, ceiling=2000, text="detail",
teaser=True, complete="stated", filters=Filters(country="ids"),
places="many", fields=Fields(posted="listed", remote="stated"),
empty="suspect", title="provisional")
assert readers.problems(module, "fake") == []
def test_a_module_without_reads_is_checked_as_before() -> None:
module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: [])
assert readers.problems(module, "fake") == []
assert readers.scope(module) == "board"
assert readers.undeclared({"fake": module}) == ["fake"]
@pytest.mark.parametrize(("module", "reads", "problem"), [
(_reader(feed_url=lambda slug, ask: "", parse=lambda b, s: []),
{"listing": "paged", "complete": "stated"}, 'listing="paged" without page and rows'),
(_paged(), {}, 'listing="whole" without feed_url and parse'),
(_whole(), {"complete": "stated"}, 'complete="whole" goes with listing="whole"'),
(_whole(), {"page_size": 20}, 'page_size and ceiling are for listing="paged"'),
(_paged(), {"listing": "paged", "complete": "end", "page_size": 100, "ceiling": 50},
"ceiling is below page_size"),
(_paged(fallback_url=lambda slug: ""), {"listing": "paged", "complete": "end"},
'fallback_url without listing="whole"'),
(_whole(), {"text": "detail"}, 'text="detail" without detail'),
(_whole(detail=lambda g, s, r: {}), {}, 'detail with text="listing"'),
(_whole(), {"teaser": True, "text": "excerpt"}, "teaser is for text=\"detail\""),
(_whole(), {"title": "provisional"}, 'title="provisional" without detail'),
(_whole(), {"filters": Filters(country="ids")}, "an ids filter (country) without facets"),
(_whole(facets=lambda g, s: {}), {"filters": Filters(country="one")},
"facets without an ids filter"),
(_whole(facets=lambda g, s: {}), {"filters": Filters(query="ids")},
'query cannot be "ids"'),
(_paged(), {"scope": "search", "listing": "paged", "complete": "end"},
'scope="search" needs complete="never" or "stated"'),
(_paged(), {"listing": "paged", "complete": "never"},
'scope="board" cannot be complete="never"'),
(_whole(), {"key": Key(name="Adzuna Key", fields=("app_id",))},
"key names no config entry"),
(_whole(), {"key": Key(name="adzuna", fields=())}, "key has no fields"),
(_whole(), {"key": Key(name="adzuna", fields=("app_id", "app_key"),
env=("ADZUNA_APP_ID",))},
"key.env does not match key.fields"),
(_whole(), {"robots": " "}, "robots is empty"),
(_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {},
"feed_url does not take (slug, ask)"),
(_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []),
{"listing": "paged", "complete": "end"}, "page does not take (get, slug, cursor, ask)"),
])
def test_the_declaration_check_names_each_problem(
module: Any, reads: dict[str, Any], problem: str) -> None:
module.READS = _reads(**reads)
found = readers.problems(module, "fake")
assert any(p.startswith(problem) for p in found), found
def test_reads_that_is_not_a_reads_is_refused() -> None:
module = _whole()
module.READS = {"scope": "board"}
assert "READS is not a Reads" in readers.problems(module, "fake")
def test_a_search_needs_fewer_names_than_a_board() -> None:
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL = "fake", "Fake"
module.allowed = lambda slug: lambda host: True
module.page = lambda get, slug, cursor, ask: (None, [], 0, None)
module.rows = lambda decoded, slug: []
module.READS = _reads(scope="search", listing="paged", complete="never",
filters=Filters(query="one"),
key=Key(name="adzuna", fields=("app_id", "app_key"),
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")))
assert readers.problems(module, "fake") == []
assert readers.scope(module) == "search"
del module.READS
assert "no careers_url" in readers.problems(module, "fake")
def test_a_misspelt_trait_fails_where_it_is_declared() -> None:
with pytest.raises(pydantic.ValidationError):
_reads(listing="pages")
with pytest.raises(pydantic.ValidationError):
Filters(country="some")
with pytest.raises(pydantic.ValidationError):
_reads(robots=None)
with pytest.raises(pydantic.ValidationError):
Reads(scope="board", listing="whole", text="listing", complete="whole",
places="one") # type: ignore[call-arg] # robots is required
with pytest.raises(pydantic.ValidationError):
_reads(colour="blue") # extra keys are refused
def test_a_declaration_is_frozen() -> None:
reads = _reads()
with pytest.raises(pydantic.ValidationError):
reads.places = "many" # type: ignore[misc]
def test_the_loader_refuses_a_module_whose_declaration_breaks_the_contract(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
package = tmp_path / "readersbad"
package.mkdir()
(package / "__init__.py").write_text("")
(package / "odd.py").write_text(
"from plugins.readers import Reads\n"
"NAME = 'odd'\nLABEL = 'Odd'\nPATTERNS = ()\n"
"def slug_ok(slug): return True\n"
"def allowed(slug): return lambda h: True\n"
"def careers_url(slug): return ''\n"
"def feed_url(slug, ask): return ''\n"
"def parse(body, slug): return []\n"
"READS = Reads(scope='board', listing='whole', text='detail', complete='whole',\n"
" places='many', robots='read (checked 2026-09-30)')\n")
monkeypatch.syspath_prepend(str(tmp_path))
with pytest.raises(ImportError, match=r'readersbad\.odd .*text="detail" without detail'):
readers.load([str(package)], "readersbad")
def test_filters_list_what_is_filtered() -> None:
assert Filters(country="one", query="many").declared() == {"country": "one",
"query": "many"}
# ------------------------------------------------------------ the fixture check
def _fake_many(rows: list[dict[str, Any]]) -> Any:
module = _reader(feed_url=lambda slug, ask: "", parse=lambda body, slug: rows)
module.READS = _reads(places="many")
return module
def test_the_place_check_passes_a_role_at_two_places() -> None:
rows = [{"source_id": "1", "places": [{"text": "Berlin", "remote": False},
{"text": "Remote (DE)", "remote": True}]}]
assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == []
def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None:
joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}]
assert place_problems(joined, "many") == [
'no row has two places: a reader with places="many" keeps every place']
assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [
"2: places is not a list"]
assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [
"3: ('Berlin', False) is not a Place"]
def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
seen: list[bytes] = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
seen.append(get("https://fake.example/list"))
return ([{"id": "1"}], [{"id": "1"}], 1, None)
module = _paged(page=page, rows=lambda decoded, slug: [{"source_id": d["id"], "places": []}
for d in decoded],
detail=lambda get, slug, row: {"places": [
{"text": get("https://fake.example/1").decode(), "remote": None},
{"text": "Munich", "remote": None}]})
module.READS = _reads(listing="paged", complete="stated", text="detail", places="many")
rows = read_fixture(module, b"Berlin", "acme", detail_body=b"Berlin")
assert seen == [b"Berlin"]
assert place_problems(rows, "many") == []
def test_the_contract_types_are_what_the_package_exports() -> None:
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}