Merge branch 'wp0-hosts'

This commit is contained in:
Wolf committed 2026-09-29 19:50:12 +02:00
commit d4fc1ac487
20 files changed
+2760 -913

No files matched your search

+97 -88
View File
@@ -5,10 +5,12 @@ The seed list and the LinkedIn flow both start from a company name and a
website, not from a board. This module gets from the one to the other: it
fetches the site's careers page, or the pages the home page links to as
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
host (one the `ats` plugin parses) becomes a `board` target the pull can
read; any other host is recorded as `ats`, with its slug where the URL
carries one, so the company is kept as one the app cannot read yet rather
than dropped.
host (one with a module in `plugins/atshosts/`, which the `ats` plugin
parses) becomes a `board` target the pull can read; any other host is
recorded as `ats`, with its slug where the URL carries one, so the company is
kept as one the app cannot read yet rather than dropped. The patterns and
probes of a readable host are its module's; the hosts only named are in
`_UNREADABLE` here, and one of them that gains a module is read from then on.
A Teamtailor site on the company's own domain names no slug: the page only
loads Teamtailor's CDN. The board there is the site's own host, and it is
@@ -16,7 +18,8 @@ taken only once that host answers `/jobs.rss` with a Teamtailor feed.
Only when no page names a host at all are the hosts with a cheap per-slug
endpoint (`_PROBES`) probed with the company's own slug, because a site that
draws its jobs from an iframe or a script leaves no link to find. A page that
draws its jobs from an iframe or a script leaves no link to find. Lever is
probed on its EU instance as well, as `lever:eu/<slug>`. A page that
names an unreadable host is the answer: a probe on a common name can hit
another tenant's board.
@@ -37,13 +40,15 @@ from __future__ import annotations
import logging
import re
from collections.abc import Callable, Iterable
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, replace
from types import ModuleType
from urllib.parse import urljoin
import httpx
from desk.sources import USER_AGENT
from plugins import atshosts
log = logging.getLogger(__name__)
@@ -53,55 +58,61 @@ MAX_BYTES = 2 << 20
MAX_PAGES = 6
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio", "teamtailor")
READABLE: tuple[str, ...] = tuple(atshosts.HOSTS)
#: host -> pattern over a URL. Group 1, where there is one, is the slug the
#: host names the company by; a host with no group is recognised but its
#: board has no slug the app could use. Order matters only for readability:
#: the readable hosts come first so the first match on a page is the one the
#: app can read.
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
(host, re.compile(pattern, re.IGNORECASE)) for host, pattern in (
("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"),
("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"),
("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"),
("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"),
("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"),
("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"),
("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"),
("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"),
("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"),
("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"),
("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"),
("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"),
# No slug: a career site on the company's own domain, which only loads
# Teamtailor's CDN. `_discover` reads its board off the page's host.
("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"),
# Not readable. The slug, where the URL has one, is kept for the day
# a parser exists.
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
("eightfold", r"\.eightfold\.ai"),
("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"),
("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
))
#: host -> pattern over a URL, for the hosts discovery names but the app
#: cannot read. Group 1, where there is one, is the slug the host names the
#: company by, kept for the day a parser exists; a host with no group is
#: recognised but its board has no slug the app could use.
_UNREADABLE: tuple[tuple[str, str], ...] = (
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
("eightfold", r"\.eightfold\.ai"),
("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"),
("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
)
def _hosts(modules: Mapping[str, ModuleType],
unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]:
"""Every host's patterns, compiled: the readable hosts' from their
modules first, so the first match at one place on a page is the one the
app can read, then the unreadable ones. An unreadable host that has
gained a module is left out here, since its module's patterns replace
it."""
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
#: host -> pattern over a URL, every host discovery can name. The slug is
#: group 1, or, for a pattern with more groups, the groups that matched
#: joined by "/" (Lever's `eu/<site>`).
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE)
#: The hosts whose slugs keep the case the URL wrote them in, by their
#: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its
#: listing and its robots.txt paths are case-sensitive. Every other slug is
#: lowercased, which is how a page that writes `BoschGroup` and one that
#: writes `boschgroup` name one board.
_KEEP_CASE = frozenset(name for name, module in atshosts.HOSTS.items()
if getattr(module, "KEEP_CASE", False))
#: What a Teamtailor feed carries and a page that only looks like one does not:
#: the namespace of `tt:locations`, declared on `<rss>` even when no role is open.
@@ -125,7 +136,7 @@ _ANCHOR = re.compile(r"<a\b[^>]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)
re.IGNORECASE | re.DOTALL)
_TAGS = re.compile(r"<[^>]+>")
#: Words a pattern's slug group can catch that name no board: the hosts'
#: Words a pattern's group 1 can catch that name no board: the hosts'
#: own path segments, the analytics subdomain Recruitee sites embed, and
#: the CDN and the admin app Teamtailor sites load from (cdn.teamtailor.com,
#: app.teamtailor.com).
@@ -161,9 +172,13 @@ def boards_in(text: str) -> list[Found]:
hits: list[tuple[int, str, str, str]] = []
for host, pattern in HOSTS:
for m in pattern.finditer(text):
slug = (m.group(1) if m.groups() and m.group(1) else "").lower()
if slug in _NOT_SLUGS:
# Only group 1 names a board or a host's own path word; a later
# group may be anything, a Workday site called `jobs` included.
if m.re.groups and (m.group(1) or "").lower() in _NOT_SLUGS:
continue
slug = "/".join(part for part in m.groups() if part)
if host not in _KEEP_CASE:
slug = slug.lower()
hits.append((m.start(), host, slug, m.group(0)))
for _pos, host, slug, url in sorted(hits):
if (host, slug) in seen:
@@ -237,25 +252,28 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
return "", ""
#: host -> a cheap URL that answers 2xx only when the slug is a board there.
#: Workable is left out: its API sits behind a rate limit that a run of
#: guesses trips, and a Workable site links its board plainly anyway.
_PROBES = {
"greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}",
"lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json",
"ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}",
"smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1",
"recruitee": "https://{slug}.recruitee.com/api/offers/",
"personio": "https://{slug}.jobs.personio.de/xml",
"teamtailor": "https://{slug}.teamtailor.com/jobs.rss",
}
#: host -> a cheap URL that answers 2xx only when the slug is a board there,
#: from each module's `probe_url`. Workable has none: its API sits behind a
#: rate limit that a run of guesses trips, and a Workable site links its
#: board plainly anyway.
_PROBES: dict[str, Callable[[str], str]] = {
name: module.probe_url for name, module in atshosts.HOSTS.items()
if hasattr(module, "probe_url")}
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
"""The slugs a guessed name is probed as on `host`: the name itself, and
for Lever the name on its EU instance too (the module's `probe_slugs`)."""
module = atshosts.HOSTS[host]
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
def probe(client: httpx.Client, host: str, slug: str) -> bool:
"""Whether `host` has a board called `slug`, by its cheapest endpoint.
Ashby and SmartRecruiters answer an empty listing for an unknown slug,
so there a listing without jobs counts as no board."""
url = _PROBES[host].format(slug=slug)
A host that answers an unknown slug with an empty listing (Ashby,
SmartRecruiters, ...) says in its module's `probe_ok` what a listing
with a board looks like."""
url = _PROBES[host](slug)
try:
response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
headers={"User-Agent": USER_AGENT})
@@ -263,18 +281,8 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool:
return False
if not response.is_success:
return False
body = response.content
if host == "ashby":
return b'"jobs":[' in body and b'"jobs":[]' not in body
if host == "smartrecruiters":
return b'"totalFound":' in body and b'"totalFound":0' not in body
if host == "recruitee":
return b'"offers":[' in body and b'"offers":[]' not in body
if host == "personio":
return b"<position>" in body
if host == "teamtailor":
return b"<item>" in body
return True
module = atshosts.HOSTS[host]
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
Fetch = Callable[[httpx.Client, str], tuple[str, str]]
@@ -377,9 +385,10 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
# probe could only add another tenant's board of the same name.
if unreadable is not None:
return unreadable
for slug in slug_guesses(name, site):
for guess in slug_guesses(name, site):
for host in _PROBES:
if probe_slug(client, host, slug):
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
careers_url=careers_page, evidence=_PROBES[host].format(slug=slug))
for slug in _probe_slugs(host, guess):
if probe_slug(client, host, slug):
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
careers_url=careers_page, evidence=_PROBES[host](slug))
return Discovery(careers_url=careers_page)
+5 -1
View File
@@ -111,8 +111,12 @@ def detach_board(store: Store, key: str) -> int:
def _forget_failing(store: Store, target: str) -> None:
"""Drop what the pull recorded about a board nobody reads any more: a
record of `target` failing would otherwise outlive the board, and count
in `app companies check` for good."""
in `app companies check` for good. The `ats` plugin's robots.txt verdict
and its lists of roles read and failed in detail go with it, since they
are about that board alone and would only grow stale."""
store.state.pop(FAILING + target, None)
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
store.state.pop(key + target, None)
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
+1 -1
View File
@@ -661,7 +661,7 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
#: read as when a role was posted, checked against live boards on
#: 2026-09-29:
#:
#: - `greenhouse`: `plugins.ats._parse_greenhouse` already prefers
#: - `greenhouse`: `plugins.atshosts.greenhouse.parse` already prefers
#: `first_published` over `updated_at`. A live board (adyen) showed a job
#: with `first_published` five months before `updated_at`, proving the
#: first stays fixed while the second moves on an edit — the resettable
+34 -14
View File
@@ -46,32 +46,52 @@ The 49 were missed because the slug is not the company's name (`BoschGroup`, `ji
## Contracts between packages
Host module, `plugins/atshosts/<host>.py`:
Host module, `plugins/atshosts/<host>.py`, as built by WP0. The docstring of `plugins/atshosts/__init__.py` holds the same contract and is the one to keep current. A module imports `desk` and `._shared` only, never `plugins.ats`.
```python
NAME = "workday" # the host of a target `workday:<slug>`
LABEL = "Workday" # the Sources page's select
PATTERNS = (r"https?://...",) # discover: over a URL, group 1 is the slug
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's file name
LABEL = "Workday" # the Sources page's select
PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int, default 100; lower comes first
KEEP_CASE = True # optional, default False; keep a discovered slug's case
def slug_ok(slug: str) -> bool: ... # strict; a slug that fails is no target
def allowed(slug: str) -> Callable[[str], bool]: ... # hostnames a fetch may reach
def careers_url(slug: str) -> str: ... # the page a person opens
def robots_page(slug: str) -> str: ... # optional; the URL robots.txt must allow
def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch
def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach
def careers_url(slug: str) -> str: ... # the page a person opens
# one of the two
def feed_url(slug: str) -> str: ...
def feed_url(slug: str) -> str: ... # the whole board in one answer
def parse(body: bytes, slug: str) -> list[Row]: ...
def page(get: Get, slug: str, cursor: Any) -> Page: ...
def page(get: Get, slug: str, cursor: Any) -> Page: ... # a paged board, one page a call
def rows(decoded: Any, slug: str) -> list[Row]: ...
# optional
def detail(get: Get, slug: str, row: Row) -> Row: ...
def probe_url(slug: str) -> str: ...
def probe_ok(body: bytes) -> bool: ...
def fallback_url(slug: str) -> str: ... # feed hosts: a smaller feed, read once when feed_url is over the byte cap
def robots_page(slug: str) -> str: ... # the URL robots.txt must allow; on a host allowed(slug) admits
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: ... # fields from the role's own page
def probe_url(slug: str) -> str: ... # discovery: answers 2xx only for a board
def probe_ok(body: bytes) -> bool: ... # discovery: when 2xx is not enough
def probe_slugs(guess: str) -> tuple[str, ...]: ... # the slugs a guessed name is probed as; default (guess,)
```
`Get` is `get(url, *, method="GET", payload=None, headers=None) -> bytes`. `Row` and `Page` are as in `plugins/ats.py` today. `plugins/atshosts/__init__.py` lists the modules with `pkgutil.iter_modules`, sorted by name, so a new host registers by existing.
`Get`, `Row` and `Page` are in `plugins/atshosts/_shared.py`, with the helpers several hosts share: `only(host)` for `allowed`, `path_slug_ok` and `label_slug_ok` for `slug_ok`, `parse_dt`, `is_web_url`, `num`, `yearly_amount` and `pay_line`. `plugins/atshosts/__init__.py` lists the modules with `pkgutil.iter_modules`, checks each against the contract at import (an ImportError names the module and what it lacks), and orders `HOSTS` by `(RANK, NAME)`. The eight first hosts hold ranks 10 to 80, which keeps the order they had before the modules: the select opens on Greenhouse, and a guessed slug on two hosts goes to the one probed first.
Order and names. `RANK` orders the Sources page's select, the plugin's description and discovery's probes. `NAME` equals the file name. A host that `discover.py` names in its own `_UNREADABLE` table is taken from its module as soon as the module exists, so a wave 1 package does not edit `discover.py`.
Patterns. The slug is group 1, or, when a pattern has more groups, the groups that matched joined by "/" (`https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)` gives `lego.wd103/LEGO_External`). Only group 1 is checked against discovery's words that name no board (`api`, `jobs`, `embed`, ...). The slug is lowercased unless the module sets `KEEP_CASE = True`, which a host with case-sensitive paths needs, since robots.txt paths are case-sensitive too. A pattern without a group names the host with no slug.
Slugs. `slug_ok` uses `re.fullmatch` (a `$` lets a trailing newline through) and refuses anything that could add a path, query, fragment, port or user to the URLs built from it. A slug of several parts joins them with "/": Lever's EU boards are `lever:eu/<site>`.
`get(url, *, method="GET", payload=None, headers=None) -> bytes` is `ats._get` bound to the board. A POST sends `payload` as JSON; `headers` are added to the default User-Agent and Accept and replace either where they name it. It fetches https on port 443 only, never an IP address or a local name (`localhost`, `.local`, `.internal`, `.lan`, `.home.arpa`), follows at most three redirects each checked against `allowed(slug)`, cuts the body at `MAX_BYTES` and ends within `DEADLINE`. It returns the body and raises `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError` on a timeout or a connection failure, and another exception, `ValueError` among them, for a refused host, an oversized body or a broken redirect. A module lets these through.
`Page` is `(decoded, results, total, cursor)`: the decoded page handed to `rows`, its raw results as dicts with an `id` (counted by distinct id for completeness), the total the host states or None, and the next cursor or None after the last page. The first call gets cursor None. A board with a total is complete when as many distinct ids were read, one without when the read ended on an empty page; a 429 after the first page ends the read, incomplete. A paged module may also have `parse(body, slug)` for one page's body, for its tests.
A `Row` has `source_id`, `title`, `location`, `url`, `posted` (aware datetime or None), `department`, `description` (plain text), `remote` (True, False, a word or None) and `salary_text`, and, where the host states pay, `salary_min`, `salary_max` (yearly integers) and `currency`.
`detail` returns a dict of what the role's own page states. The shared code calls it for roles not yet read (`ats:detailed:<target>`), at most `DETAIL_CAP` (200) per board per pull, `PAGE_DELAY` apart, the roles never tried before those whose detail failed (`ats:detailfailed:<target>`). Only `DETAIL_FIELDS` are taken: `description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the listing's empty value (`upsert_role`'s keep-filled columns, the salary figures, and `posted_at`, which keeps the earliest date). Any other key is dropped with a debug log line: `title`, `url`, `department` and `salary_text` cannot be filled from `detail`, since the listing rewrites them every pull. A role not read in detail on a pull is sent with an empty description. A 404 or 410 counts as read, any other failure is retried on a later pull, and a 429 ends the detail reads for that board and pull. Completeness is decided by the listing alone.
robots.txt. For a module with `robots_page`, the shared code reads `https://<its host>/robots.txt` before the listing, at most once a day per board (`ats:robots:<target>`), following a redirect to the host's `www.` twin or back and no other. It fails the board with "robots.txt disallows it" when the rules for `*` or for the app's own token disallow the page, matched per RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a final `$` in paths, paths case-sensitive, an empty Disallow allows. A 4xx other than 429 means there are no rules, which allows. A 5xx, a 429, a timeout, a connection error or a redirect elsewhere fails the board with "robots.txt could not be read", and the next pull asks again.
Slugs: Workday `lego.wd103/LEGO_External`; SuccessFactors, Phenom and the sitemap reader take the site's hostname, with a path where the site has one (`jobs.teradyne.com/MiR`); Oracle `<host>/<siteNumber>`.
+7 -5
View File
@@ -1,8 +1,10 @@
"""Drop-in plugins, discovered by `plugin_paths` in `config.yaml`.
A plugin file imports `desk` only — see `tests/test_architecture.py`. This
`__init__.py` makes the directory importable as `plugins.<name>` for the rest
of roledesk (`jobs/pull.py`, `cli.py`, the tests), which the plugin contract
itself never relies on: `desk.sources.registry` finds `ats.py` by path, not
by this package.
A plugin file imports `desk` only — see `tests/test_architecture.py` — and
its own helpers in this directory. This `__init__.py` makes the directory
importable as `plugins.<name>`, for the rest of roledesk (`jobs/pull.py`,
`cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the
package `plugins/atshosts/`. `desk.sources.registry` still finds `ats.py` by
path, not by this package; `ats.py` puts the repo root on `sys.path` itself
when the app has not, so that `plugins.atshosts` imports either way.
"""
+440 -789
View File
File diff suppressed because it is too large. Load diff
+210
View File
@@ -0,0 +1,210 @@
"""The hosts the `ats` plugin reads, one module each.
A host registers by existing: every module in this package whose name does
not start with `_` is loaded when the package is imported, checked against
the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A
module that breaks the contract fails the import with an ImportError naming
the module and what it lacks, rather than a pull that fails board by board.
`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host
select and its description from `HOSTS`, and `app/jobs/discover.py` its
patterns and probes. The package sits beside `ats.py` rather than being
`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and
the tests patch names on the `ats` module.
A module is named after its host (`plugins/atshosts/workday.py`) and imports
`desk` and `._shared` only, never `plugins.ats`. It holds:
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
LABEL = "Workday" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int; lower comes first in the select and the probes
KEEP_CASE = False # optional; True keeps a discovered slug's case
def slug_ok(slug: str) -> bool # strict, see below
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
def careers_url(slug: str) -> str # the page a person opens
and one of two ways to read a board:
def feed_url(slug: str) -> str # the whole board in one answer,
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call,
def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page
and, optionally:
def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read
# once when `feed_url` is over the cap
def robots_page(slug: str) -> str # the URL robots.txt must allow
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page
def probe_url(slug: str) -> str # discovery: 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
hold 10 to 80, which keeps the order they had before there were modules, so
a new host comes after them unless it says otherwise. The order decides the
select's default and which host wins when discovery's guessed slug exists on
two.
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
which lets a trailing newline through) and refuses anything that could add a
path, a query, a fragment, a port or a user to the URLs built from it.
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
(one DNS label) cover the usual cases; a slug with several parts joins them
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
**Patterns.** The slug is group 1, or, when a pattern has more than one
group, the groups that matched joined by "/". Only group 1 is checked
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
paths are case-sensitive (Workday's site), since robots.txt paths are too. A
pattern without a group names the host with no slug. A host that
`discover.py` names in its own `_UNREADABLE` table is taken from its module
instead as soon as the module exists.
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
name; `_shared.only(host)` is the usual one. Every request goes through
`get`, which is `ats._get` bound to the board:
get(url, *, method="GET", payload=None, headers=None) -> bytes
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
default User-Agent and `Accept: application/json, text/xml`, and replace
either where they name it. `get` is https only on port 443, never to an IP
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
most three redirects each checked against `allowed(slug)`, cuts the body at
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
failure, and another exception (`ValueError` among them) for a refused host,
an oversized body or a broken redirect. A module lets these through; the
shared code logs, records and decides.
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
id, to decide completeness), the total the host states or None, and the
cursor of the next page or None after the last. The first call gets cursor
None. A board with a total is complete when as many distinct ids were read;
one without, when the read ended on an empty page. A 429 after the first page
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
apart. A paged module may also have `parse(body, slug)` for one page's body.
**Rows.** A `Row` is a dict with `source_id` (the host's own id for the
role, unique on the board), `title`, `location`, `url` (http or https,
checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see
`_shared.parse_dt`), `department`, `description` (plain text, see
`desk.sources.strip_html`), `remote` (True, False, a word or None) and
`salary_text`; and, where the host states pay, `salary_min`, `salary_max`
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`.
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
dict of the fields it found. The shared code calls it for roles not yet read
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
and `posted`, the keys whose columns survive a later pull that sends the
listing's empty value; any other key is left out with a debug log line.
`title`, `url`, `department` and `salary_text` cannot be filled from
`detail`: the listing rewrites them every pull. A role not read in detail
on a pull is sent with an empty description, so the one read earlier stays.
A 404 or a 410 counts as read; any other failure is logged and retried on a
later pull; a 429 ends the detail reads of that board for the pull.
Completeness is decided by the listing alone.
**robots.txt.** With `robots_page(slug)`, the shared code reads
`https://<its host>/robots.txt` through `get` before the listing, at most
once a day per board (`ats:robots:<target>`), following a redirect to the
host's `www.` twin or back and no other. The board fails with "robots.txt
disallows it" when the rules for `*` or for the app's own token disallow the
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
final `$` in paths, case-sensitive paths). A 4xx other than 429 means no
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
the board with "robots.txt could not be read" and is asked again next pull.
`robots_page` must be on a host `allowed(slug)` admits.
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
the slug is a board; `probe_ok(body)` narrows that when a host answers an
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
name is probed as, `(guess,)` without it. A host without `probe_url` is never
probed, only found by a link.
"""
from __future__ import annotations
import importlib
import pkgutil
import re
from collections.abc import Iterable
from types import ModuleType
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
DEFAULT_RANK = 100
#: What every host module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal."""
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
patterns = getattr(module, "PATTERNS", ())
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
out.append("PATTERNS is not a tuple of strings")
else:
for pattern in patterns:
try:
re.compile(pattern)
except (TypeError, re.error) as exc:
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
out += [f"{attr} is not callable" for attr in FUNCTIONS
if hasattr(module, attr) and not callable(getattr(module, attr))]
if hasattr(module, "page"):
if not hasattr(module, "rows"):
out.append("page without rows")
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
out.append("neither feed_url and parse nor page and rows")
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
out.append("probe_ok without probe_url")
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
out.append("probe_slugs without probe_url")
return out
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
"""`NAME -> module` for every module of `package` (found on `path`) whose
name does not start with `_`, in order of `(RANK, NAME)`. Raises
ImportError naming the first module that breaks the contract."""
modules: list[ModuleType] = []
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{package}.{info.name}")
if found := problems(module, info.name):
raise ImportError(f"{package}.{info.name} breaks the host contract: "
+ "; ".join(found))
modules.append(module)
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
return {module.NAME: module for module in modules}
#: host -> its module, in order of `(RANK, NAME)`.
HOSTS: dict[str, ModuleType] = load(__path__, __name__)
+192
View File
@@ -0,0 +1,192 @@
"""What the host modules share: the row and page shapes, the fetch they are
handed, and the date, URL and salary helpers more than one host needs.
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
import it without either importing the other. The leading underscore keeps
`plugins.atshosts` from listing it as a host.
"""
from __future__ import annotations
import re
from collections.abc import Callable
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from typing import Any, Protocol
import httpx
#: What every parser turns a feed into: one dict per row, with the keys
#: `source_id`, `title`, `location`, `url`, `posted`, `department`,
#: `description`, `remote` and `salary_text`, and, where a host states pay,
#: `salary_min`, `salary_max` and `currency`.
Row = dict[str, Any]
#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded
#: page, its raw results (dicts with an `id`, counted for completeness), the
#: total it states (None for a host that states none), and the cursor of the
#: next page (None after the last). The first call gets cursor None.
Page = tuple[Any, list[dict[str, Any]], Any, Any]
class Get(Protocol):
"""The fetch a host is handed: `ats.py`'s own, bound to one board, so the
allow-list, the redirect rule, the byte cap and the deadline apply to
every request a host makes. A POST sends `payload` as JSON; `headers` are
added to (and may replace) the default User-Agent and Accept. Returns the
body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see
"Fetching" in the package's docstring for the rest."""
def __call__(self, url: str, *, method: str = "GET",
payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes: ...
def is_web_url(url: str) -> bool:
"""A trimmed http(s) URL, or refuse — a feed is external content and a
`javascript:` or `data:` link must never reach an href. Port of
openings.go's `webURL`."""
url = (url or "").strip()
if not url:
return False
parsed = httpx.URL(url)
return parsed.scheme in ("http", "https") and bool(parsed.host)
def only(host: str) -> Callable[[str], bool]:
"""The allow-list of a board that is fetched from one host name only,
which is every host so far. A host name has no case, and a slug that
names one (`Acme` of acme.recruitee.com) may be written with capitals."""
want = host.lower()
return lambda h: h.lower() == want
#: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`).
#: A slug that is one URL path segment: what the hosts that put the slug in
#: the path accept, and nothing that could add a path, a query or a fragment.
_SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+")
#: A slug that is one DNS label, for the hosts that put it in the host name.
_LABEL = re.compile(r"[A-Za-z0-9-]+")
def path_slug_ok(slug: str) -> bool:
return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..")
def label_slug_ok(slug: str) -> bool:
return bool(_LABEL.fullmatch(slug))
def parse_dt(value: Any) -> datetime | None:
"""An aware datetime from whatever a board sends: ISO-8601 with an
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
if value is None:
return None
if isinstance(value, int | float):
if value <= 0:
return None
seconds = value / 1000 if value > 10**12 else value
return datetime.fromtimestamp(seconds, tz=UTC)
text = str(value).strip()
if not text:
return None
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
if not match:
try:
parsed = parsedate_to_datetime(text)
except (TypeError, ValueError):
return None
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
# ---------------------------------------------------------- structured pay
#
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
# `raw` into, per docs/decisions.md's "raw is transport, columns are
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
# structured pay on their list endpoint; Workable, SmartRecruiters and
# Teamtailor do not (checked live, 2026-09-29), so their rows leave the
# columns out and `ATSSource.fetch` reads a plain '' or None for them.
def num(value: Any) -> float | None:
"""A positive number from whatever a board sends (int, float, or a
numeral string), or None for anything else, including zero and a
negative — a board's placeholder 0 (an unfilled field, not a real
figure) must not become a salary floor of zero."""
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if number > 0 else None
def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
None, "")` when there is no currency, no bound, or `interval` is not one
of `factors`' own keys — which is how an hourly or a daily rate is left
out rather than annualised into a number nobody stated. `factors` maps
the board's own interval spelling to the multiplier that makes it
yearly: 1 for a figure already yearly, 12 for a monthly one."""
code = str(currency or "").strip().upper()
factor = factors.get(str(interval or "").strip())
if not code or not factor:
return None, None, ""
lo, hi = num(min_value), num(max_value)
if lo is None and hi is None:
return None, None, ""
return (int(lo * factor) if lo is not None else None,
int(hi * factor) if hi is not None else None, code)
# ------------------------------------------------------------- salary_text
#
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
# hourly and daily rates, a Greenhouse role with several pay zones, and a
# range with no currency are all dropped from those columns rather than
# guessed into a wrong yearly figure. `salary_text` is where that dropped
# information survives — one human-readable line per parser, written
# whether or not the pay conformed, so a person can read it and do their own
# sums and a stricter analysis can find the roles the columns left out
# (`Role.salary_state`, `app/models.py`).
#: Formats a number the way a salary line writes it: no decimals for a whole
#: figure (the normal case for a yearly or monthly one), two for anything
#: else (an hourly or daily rate quoted with cents).
def fmt_amount(value: float) -> str:
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
def bounds_words(lo: float | None, hi: float | None) -> str:
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
bound stated, or "" with neither."""
if lo is not None and hi is not None:
return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}"
if lo is not None:
return f"from {fmt_amount(lo)}"
if hi is not None:
return f"up to {fmt_amount(hi)}"
return ""
def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
per year" — or "" when there is no bound to report at all. A currency
that was not stated does not drop the line, unlike the normalised
columns: it is said plainly instead, "80,000 to 100,000 per year (no
currency stated)", so the figure is not lost along with the currency."""
bounds = bounds_words(lo, hi)
if not bounds:
return ""
tail = f" per {period}" if period else ""
code = str(currency or "").strip().upper()
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
+122
View File
@@ -0,0 +1,122 @@
"""Ashby: `ashby:<job board name>`, the whole board in one JSON answer. A
board name may carry a dot (`jimdo.com`) or be a uuid.
The field map is ported from `openings.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
NAME = "ashby"
LABEL = "Ashby"
RANK = 30
PATTERNS = (
r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)",
r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)",
)
_API = "api.ashbyhq.com"
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_API)
def careers_url(slug: str) -> str:
return f"https://jobs.ashbyhq.com/{slug}"
def feed_url(slug: str) -> str:
return f"https://{_API}/posting-api/job-board/{slug}?includeCompensation=true"
def probe_url(slug: str) -> str:
return f"https://{_API}/posting-api/job-board/{slug}"
def probe_ok(body: bytes) -> bool:
"""Ashby answers an empty listing for an unknown name, so a listing
without jobs counts as no board."""
return b'"jobs":[' in body and b'"jobs":[]' not in body
def _salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
"""The yearly range from `compensation.summaryComponents` (needs
`?includeCompensation=true` on the feed URL, see `feed_url`): the first
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
an hourly one is left out rather than annualised."""
for component in components:
if component.get("compensationType") != "Salary":
continue
return yearly_amount(component.get("minValue"), component.get("maxValue"),
component.get("currencyCode"), component.get("interval"),
{"1 YEAR": 1, "1 MONTH": 12})
return None, None, ""
def _salary_text(compensation: dict[str, Any]) -> str:
"""One line per `summaryComponents` entry that states a bound, labelled
by the board's own `compensationType` when more than one does (a salary
range beside a bonus range, say); with just the one, the label is left
off, the same as Greenhouse's. `interval` is "<n> <PERIOD>" ("1 YEAR",
"1 HOUR"); the last word, lowered, is the period. With nothing to build
a line from, Ashby's own prose summary (`compensationTierSummary`, else
`scrapeableCompensationSalarySummary`) is what is left to show."""
components = compensation.get("summaryComponents") or []
stated = [c for c in components if num(c.get("minValue")) is not None
or num(c.get("maxValue")) is not None]
lines = []
for component in stated:
period = str(component.get("interval") or "").split(" ")[-1].lower()
line = pay_line(num(component.get("minValue")), num(component.get("maxValue")),
component.get("currencyCode"), period)
if not line:
continue
if len(stated) > 1:
label = str(component.get("compensationType") or "").strip()
line = f"{label}: {line}" if label else line
lines.append(line)
if lines:
return "; ".join(lines)
return str(compensation.get("compensationTierSummary")
or compensation.get("scrapeableCompensationSalarySummary") or "")
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
for j in data.get("jobs") or []:
title = (j.get("title") or "").strip()
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
if not title or not is_web_url(url):
continue
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
compensation = j.get("compensation") or {}
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": (j.get("location") or "").strip(),
"url": url,
"posted": parse_dt(j.get("publishedAt")),
"department": (j.get("department") or j.get("team") or "").strip(),
"description": strip_html(description),
"remote": j.get("isRemote"),
"salary_text": _salary_text(compensation),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+139
View File
@@ -0,0 +1,139 @@
"""Greenhouse: `greenhouse:<board token>`, the whole board in one JSON answer.
The field map is ported from `openings.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
"""
from __future__ import annotations
import html
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line
NAME = "greenhouse"
LABEL = "Greenhouse"
RANK = 10
PATTERNS = (
r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)",
r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)",
)
_API = "boards-api.greenhouse.io"
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_API)
def careers_url(slug: str) -> str:
return f"https://job-boards.greenhouse.io/{slug}"
def feed_url(slug: str) -> str:
return f"https://{_API}/v1/boards/{slug}/jobs?content=true&pay_transparency=true"
def fallback_url(slug: str) -> str:
"""The feed without `content=true`, read once when the full one exceeds
the byte cap: the descriptions are lost, but the titles land rather than
the whole board being skipped."""
return f"https://{_API}/v1/boards/{slug}/jobs"
def probe_url(slug: str) -> str:
return f"https://{_API}/v1/boards/{slug}"
def _salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
on the feed URL, see `feed_url`), which the list endpoint already carries
in the same request — no per-role fetch. Only when there is exactly one
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
one per office) gives no way to tell which office the role in hand is
at, and picking one would misrepresent the columns. Greenhouse's own
ranges are cents, and yearly by convention except an hourly or a daily
one, named as such in `title` with no separate field to check — those
are skipped rather than annualised into a number nobody stated."""
if len(ranges) != 1:
return None, None, ""
entry = ranges[0]
title = str(entry.get("title") or "").lower()
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
return None, None, ""
code = str(entry.get("currency_type") or "").strip().upper()
if not code:
return None, None, ""
lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents"))
if lo is None and hi is None:
return None, None, ""
return (int(lo / 100) if lo is not None else None,
int(hi / 100) if hi is not None else None, code)
def _salary_text(ranges: list[dict[str, Any]]) -> str:
"""One line per `pay_input_ranges` entry, joined with "; " — every range
Greenhouse states, not only the single one `_salary` accepts for the
columns. With more than one, each is labelled with the board's own
`title` ("Zone 1 Pay Range: ..."), since that is the only way to tell the
ranges apart; with just the one, the label adds nothing and is left off.
The hour/day check is the same one `_salary` uses to keep those out of
the yearly columns; here it only picks the word that follows "per"."""
multi = len(ranges) > 1
lines = []
for entry in ranges:
title = str(entry.get("title") or "").strip()
low_title = title.lower()
period = ("hour" if any(w in low_title for w in ("hour", "/hr"))
else "day" if "day" in low_title else "year")
lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents"))
line = pay_line(lo / 100 if lo is not None else None,
hi / 100 if hi is not None else None,
entry.get("currency_type"), period)
if not line:
continue
if multi:
label = title.rstrip(":").strip() or "Range"
line = f"{label}: {line}"
lines.append(line)
return "; ".join(lines)
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
for j in data.get("jobs") or []:
title = (j.get("title") or "").strip()
url = (j.get("absolute_url") or "").strip()
if not title or not is_web_url(url):
continue
departments = j.get("departments") or []
# Greenhouse ships the description HTML-entity-escaped inside the
# JSON string (`&lt;p&gt;`), so it needs unescaping before the tags
# themselves can be stripped.
content = html.unescape(j.get("content") or "")
ranges = j.get("pay_input_ranges") or []
salary_min, salary_max, currency = _salary(ranges)
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": ((j.get("location") or {}).get("name") or "").strip(),
"url": url,
"posted": parse_dt(j.get("first_published") or j.get("updated_at")),
"department": (departments[0].get("name") if departments else "") or "",
"description": strip_html(content),
"remote": None, # Greenhouse's feed does not say
"salary_text": _salary_text(ranges),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+121
View File
@@ -0,0 +1,121 @@
"""Lever: `lever:<site>`, or `lever:eu/<site>` for a board on Lever's EU
instance, the whole board in one JSON answer.
Lever runs two instances that share nothing but the API: `api.lever.co` and
`api.eu.lever.co`, with the pages on `jobs.lever.co` and `jobs.eu.lever.co`.
A site exists on one of them only (TomTom and Prima answer 404 on the other,
2026-09-29), and the same name on both would be two companies, so the region
is part of the slug. It is a prefix a plain site name cannot carry, since a
Lever site name has no slash, which leaves every `lever:<site>` target as it
was.
The field map is ported from `openings.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
NAME = "lever"
LABEL = "Lever"
RANK = 20
#: Two groups on the EU hosts, `eu` and the site, which discovery joins into
#: the slug `eu/<site>`.
PATTERNS = (
r"jobs\.lever\.co/([A-Za-z0-9_-]+)",
r"api\.lever\.co/v0/postings/([A-Za-z0-9_-]+)",
r"jobs\.(eu)\.lever\.co/([A-Za-z0-9_-]+)",
r"api\.(eu)\.lever\.co/v0/postings/([A-Za-z0-9_-]+)",
)
#: The region prefixes a slug may carry, to the infix of both host names.
_REGIONS = {"eu": "eu."}
def _split(slug: str) -> tuple[str, str]:
"""`(host infix, site)`: `("eu.", "tomtom")` for `eu/tomtom`, `("", "acme")`
for `acme`."""
region, sep, site = slug.partition("/")
return (_REGIONS.get(region, "?"), site) if sep else ("", slug)
def slug_ok(slug: str) -> bool:
infix, site = _split(slug)
return infix != "?" and path_slug_ok(site)
def allowed(slug: str) -> Callable[[str], bool]:
return only(f"api.{_split(slug)[0]}lever.co")
def careers_url(slug: str) -> str:
infix, site = _split(slug)
return f"https://jobs.{infix}lever.co/{site}"
def feed_url(slug: str) -> str:
infix, site = _split(slug)
return f"https://api.{infix}lever.co/v0/postings/{site}?mode=json"
def probe_url(slug: str) -> str:
infix, site = _split(slug)
return f"https://api.{infix}lever.co/v0/postings/{site}?limit=1&mode=json"
def probe_slugs(guess: str) -> tuple[str, ...]:
"""A name is tried on both instances, the US one first, since a site that
names neither leaves no other way to tell which one it is on."""
return (guess, *(f"{region}/{guess}" for region in _REGIONS))
def _salary_text(salary_range: dict[str, Any]) -> str:
"""One line from `salaryRange`, whatever its `interval` — including an
hourly or daily one, which `yearly_amount` drops from the columns.
Lever's interval is "per-<period>-<kind>" ("per-year-salary",
"per-hour-wage"); the middle word is the period."""
interval = str(salary_range.get("interval") or "")
parts = interval.split("-")
period = parts[1] if len(parts) >= 3 else ""
return pay_line(num(salary_range.get("min")), num(salary_range.get("max")),
salary_range.get("currency"), period)
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
if not isinstance(data, list):
return []
rows = []
for j in data:
title = (j.get("text") or "").strip()
url = (j.get("hostedUrl") or "").strip()
if not title or not is_web_url(url):
continue
categories = j.get("categories") or {}
description = j.get("description") or j.get("descriptionPlain") or ""
salary_range = j.get("salaryRange") or {}
salary_min, salary_max, currency = yearly_amount(
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": (categories.get("location") or "").strip(),
"url": url,
"posted": parse_dt(j.get("createdAt")),
"department": (categories.get("department") or categories.get("team") or "").strip(),
"description": strip_html(description),
"remote": (j.get("workplaceType") or "").strip() or None,
"salary_text": _salary_text(salary_range),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+101
View File
@@ -0,0 +1,101 @@
"""Personio: `personio:<subdomain>`, the whole board in one XML answer from
the company's own subdomain of jobs.personio.de.
Written from one real board on 2026-09-23; jobhunt-hq did not read
Personio.
"""
from __future__ import annotations
import xml.etree.ElementTree as ET
from collections.abc import Callable
from desk.sources import strip_html
from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "personio"
LABEL = "Personio"
RANK = 70
PATTERNS = (r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)",)
#: Personio's `salaryInformation/type` spellings, to the plain period word
#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through.
_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
"daily": "day", "weekly": "week"}
def slug_ok(slug: str) -> bool:
return label_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
"""The company's own subdomain, so one company's slug cannot pass for
another's."""
return only(f"{slug}.jobs.personio.de")
def careers_url(slug: str) -> str:
return f"https://{slug}.jobs.personio.de"
def feed_url(slug: str) -> str:
return f"https://{slug}.jobs.personio.de/xml"
def probe_url(slug: str) -> str:
return feed_url(slug)
def probe_ok(body: bytes) -> bool:
return b"<position>" in body
def _salary_text(salary_info: ET.Element) -> str:
"""One line from a `<salaryInformation>` element, whatever its `type`,
including an hourly or daily one that `yearly_amount` drops from the
columns."""
period_raw = (salary_info.findtext("type") or "").strip().lower()
period = _PERIODS.get(period_raw, period_raw)
return pay_line(num(salary_info.findtext("min")), num(salary_info.findtext("max")),
salary_info.findtext("currencyCode"), period)
def parse(body: bytes, slug: str) -> list[Row]:
"""`{slug}.jobs.personio.de/xml` -> `<workzag-jobs><position>...`. No job
URL in the feed itself; Personio's own page is `/job/{id}`, confirmed
live against a real posting (2026-09-23)."""
root = ET.fromstring(body)
rows = []
for position in root.findall("position"):
job_id = (position.findtext("id") or "").strip()
title = (position.findtext("name") or "").strip()
if not job_id or not title:
continue
description = " ".join(
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
salary_info = position.find("salaryInformation")
salary_min = salary_max = None
currency = ""
salary_text = ""
if salary_info is not None:
salary_min, salary_max, currency = yearly_amount(
salary_info.findtext("min"), salary_info.findtext("max"),
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
{"yearly": 1, "monthly": 12})
salary_text = _salary_text(salary_info)
rows.append({
"source_id": job_id,
"title": title,
"location": (position.findtext("office") or "").strip(),
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
"posted": parse_dt(position.findtext("createdAt")),
"department": (position.findtext("department") or "").strip(),
"description": strip_html(description),
"remote": "remote" in (position.findtext("office") or "").lower() or None,
"salary_text": salary_text,
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+84
View File
@@ -0,0 +1,84 @@
"""Recruitee: `recruitee:<subdomain>`, the whole board in one JSON answer
from the company's own subdomain of recruitee.com.
Written from one real board on 2026-09-23; jobhunt-hq did not read
Recruitee.
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "recruitee"
LABEL = "Recruitee"
RANK = 60
PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",)
def slug_ok(slug: str) -> bool:
return label_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
"""The company's own subdomain, so one company's slug cannot pass for
another's."""
return only(f"{slug}.recruitee.com")
def careers_url(slug: str) -> str:
return f"https://{slug}.recruitee.com"
def feed_url(slug: str) -> str:
return f"https://{slug}.recruitee.com/api/offers/"
def probe_url(slug: str) -> str:
return feed_url(slug)
def probe_ok(body: bytes) -> bool:
return b'"offers":[' in body and b'"offers":[]' not in body
def _salary_text(salary: dict[str, Any]) -> str:
"""One line from Recruitee's `salary` object; `period` is already a
plain word ("year", "month")."""
period = str(salary.get("period") or "").strip().lower()
return pay_line(num(salary.get("min")), num(salary.get("max")),
salary.get("currency"), period)
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
for j in data.get("offers") or []:
title = (j.get("title") or "").strip()
url = (j.get("careers_url") or "").strip()
if not title or not is_web_url(url):
continue
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
salary_min, salary_max, currency = yearly_amount(
salary.get("min"), salary.get("max"), salary.get("currency"),
salary.get("period"), {"year": 1, "month": 12})
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": (j.get("location") or j.get("city") or "").strip(),
"url": url,
"posted": parse_dt(j.get("published_at") or j.get("created_at")),
"department": (j.get("department") or "").strip(),
"description": strip_html(j.get("description") or ""),
"remote": j.get("remote"),
"salary_text": _salary_text(salary),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+90
View File
@@ -0,0 +1,90 @@
"""SmartRecruiters: `smartrecruiters:<company identifier>`, paged by offset
with a stated total. The identifier is case-sensitive (`BoschGroup`).
Written from one real board on 2026-09-23; jobhunt-hq did not read
SmartRecruiters.
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
NAME = "smartrecruiters"
LABEL = "SmartRecruiters"
RANK = 50
PATTERNS = (
r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)",
)
_API = "api.smartrecruiters.com"
#: The page size asked for, which is also the most the host serves per page.
PAGE_SIZE = 100
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_API)
def careers_url(slug: str) -> str:
return f"https://jobs.smartrecruiters.com/{slug}"
def probe_url(slug: str) -> str:
return f"https://{_API}/v1/companies/{slug}/postings?limit=1"
def probe_ok(body: bytes) -> bool:
"""An unknown identifier answers an empty listing, so a listing without
postings counts as no board."""
return b'"totalFound":' in body and b'"totalFound":0' not in body
def page(get: Get, slug: str, cursor: Any) -> Page:
offset = int(cursor or 0)
data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings"
f"?offset={offset}&limit={PAGE_SIZE}"))
content = data.get("content") or []
return data, content, data.get("totalFound"), offset + len(content) if content else None
def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
def rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
page, whatever `limit` asks for; the next is `?offset=` further on. The
list omits the job description (only the per-posting detail endpoint has
it, and fetching one per row would be hundreds of extra requests per
board), so `description` stays empty here."""
out = []
for j in data.get("content") or []:
title = (j.get("name") or "").strip()
job_id = str(j.get("id") or "")
if not title or not job_id:
continue
location = j.get("location") or {}
loc_text = (location.get("fullLocation")
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
out.append({
"source_id": job_id,
"title": title,
"location": (loc_text or "").strip(),
"url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}",
"posted": parse_dt(j.get("releasedDate")),
"department": ((j.get("department") or {}).get("label") or "").strip(),
"description": "",
"remote": location.get("remote"),
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
})
return out
+137
View File
@@ -0,0 +1,137 @@
"""Teamtailor: `teamtailor:<subdomain>` for acme.teamtailor.com, or, for a
career site on the company's own domain, that whole host name:
`teamtailor:careers.acme.com`. An RSS feed paged by offset that states no
total.
Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor.
"""
from __future__ import annotations
import html
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "teamtailor"
LABEL = "Teamtailor"
RANK = 80
PATTERNS = (
r"https?://([a-z0-9-]+)\.teamtailor\.com",
# No slug: a career site on the company's own domain, which only loads
# Teamtailor's CDN. Discovery reads its board off the page's host.
r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets",
)
#: The page size, asked for as `per_page`. A feed with neither parameter
#: answers with every role (73 of 73 at Shine, 2026-09-29), but says nowhere
#: that it is the whole list, so this reads by page to an empty one;
#: `per_page=200` was answered in full too, and the cursor advances by what a
#: page returned, so a smaller cap on the host's side would not skip a role.
PAGE_SIZE = 50
#: Only a DNS name with a letter in its last label is a site, so an address
#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could
#: be pointed at; `ats._get` still refuses every host but this one, redirects
#: included.
_HOST = re.compile(r"[a-z0-9-]+(\.[a-z0-9-]+)*")
#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`.
_TT = "{https://teamtailor.com/locations}"
#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing
#: about where the role is worked and `temporary` is remote for a spell only,
#: so both stay unknown rather than become "onsite" or "remote".
_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"}
def site(slug: str) -> str:
"""The host a career site is served from: `acme` is acme.teamtailor.com,
and a slug with a dot is a site on the company's own domain,
`careers.acme.com`, taken whole (the feed path is the same on both)."""
slug = slug.strip().lower()
return slug if "." in slug else f"{slug}.teamtailor.com"
def slug_ok(slug: str) -> bool:
host = site(slug)
return bool(slug) and slug == slug.strip() and bool(_HOST.fullmatch(host)) \
and not host.rsplit(".", 1)[-1].isdigit()
def allowed(slug: str) -> Callable[[str], bool]:
return only(site(slug))
def careers_url(slug: str) -> str:
return f"https://{site(slug)}/jobs"
def probe_url(slug: str) -> str:
return f"https://{site(slug)}/jobs.rss"
def probe_ok(body: bytes) -> bool:
return b"<item>" in body
def page(get: Get, slug: str, cursor: Any) -> Page:
offset = int(cursor or 0)
items = _items(get(f"https://{site(slug)}/jobs.rss?per_page={PAGE_SIZE}&offset={offset}"))
results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items]
return items, results, None, offset + len(items) if items else None
def parse(body: bytes, slug: str) -> list[Row]:
return rows(_items(body), slug)
def _items(body: bytes) -> list[ET.Element]:
"""`{site}/jobs.rss` -> `<rss><channel><item>...`, the feed every Teamtailor
career site serves (checked live on a dozen sites, 2026-09-29). A missing
`channel` is a page that is not a feed, and raises."""
channel = ET.fromstring(body).find("channel")
if channel is None:
raise ValueError("not a Teamtailor feed: no channel")
return channel.findall("item")
def rows(items: list[ET.Element], slug: str) -> list[Row]:
"""One page of items. The item has the title, the job URL (on the site's
own domain), a `guid` uuid, `pubDate`, the HTML `description` in full,
`remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the
`tt:` namespace, `locations` (a list: `city` and `country` each, `name` free
text), `department` and `role`, either of which may be empty. `place` joins
the distinct locations with "; ", which `country_of` reads a location at a
time. No pay is stated anywhere in the feed."""
out = []
for item in items:
title = (item.findtext("title") or "").strip()
url = (item.findtext("link") or "").strip()
if not title or not is_web_url(url):
continue
places: list[str] = []
for loc in item.findall(f"{_TT}locations/{_TT}location"):
city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip()
country = (loc.findtext(f"{_TT}country") or "").strip()
place = city if country == city else ", ".join(p for p in (city, country) if p)
if place and place not in places:
places.append(place)
out.append({
"source_id": (item.findtext("guid") or url).strip(),
"title": title,
"location": "; ".join(places),
"url": url,
"posted": parse_dt(item.findtext("pubDate")),
"department": ((item.findtext(f"{_TT}department") or "").strip()
or (item.findtext(f"{_TT}role") or "").strip()),
"description": html.unescape(strip_html(item.findtext("description") or "")),
"remote": _REMOTE.get((item.findtext("remoteStatus") or "").strip()),
"salary_text": "", # Teamtailor's feed states no pay at all
})
return out
+85
View File
@@ -0,0 +1,85 @@
"""Workable: `workable:<account>`, paged ten roles at a time with a stated
total.
Written from one real board, Hugging Face's, on 2026-09-28, when the first
board with roles was seen; jobhunt-hq did not read Workable.
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
NAME = "workable"
LABEL = "Workable"
RANK = 40
PATTERNS = (
r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)",
r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com",
)
_API = "apply.workable.com"
# No probe: Workable's API sits behind a rate limit that a run of guesses
# trips, and a Workable site links its board plainly anyway.
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_API)
def careers_url(slug: str) -> str:
return f"https://{_API}/{slug}"
def page(get: Get, slug: str, cursor: Any) -> Page:
"""One POST of the list; the next page is the same POST with
`{"token": nextPage}`, and the last page has no `nextPage`."""
url = f"https://{_API}/api/v3/accounts/{slug}/jobs"
data = json.loads(get(url, method="POST", payload={"token": cursor} if cursor else {}))
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
def rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
Face's board answered on 2026-09-28. A result carries no URL and no text:
the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and
the description is only on the per-job endpoint, one request per role,
so `description` stays empty here as it does for SmartRecruiters.
`location` is an object and `department` a list; `remote` is a flag
beside `workplace` (remote, hybrid, on_site)."""
out = []
for j in data.get("results") or []:
title = (j.get("title") or "").strip()
shortcode = (j.get("shortcode") or "").strip()
if not title or not shortcode:
continue
loc = j.get("location") or {}
location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p)
dept = j.get("department") or ""
if isinstance(dept, list):
dept = ", ".join(d for d in dept if d)
out.append({
"source_id": str(j.get("id") or shortcode),
"title": title,
"location": location,
"url": f"https://{_API}/{slug}/j/{shortcode}/",
"posted": parse_dt(j.get("published")),
"department": dept.strip(),
"description": "",
"remote": bool(j.get("remote")) or j.get("workplace") == "remote",
"salary_text": "", # Workable's list endpoint states no pay at all
})
return out
+6 -5
View File
@@ -19,6 +19,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats # noqa: E402
from plugins.atshosts import smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -384,7 +385,7 @@ def test_is_known_target_accepts_the_eight_hosts_and_refuses_others() -> None:
"recruitee", "personio", "teamtailor"):
assert ats.is_known_target(f"{host}:acme")
assert not ats.is_known_target("linkedin:acme")
assert not ats.is_known_target("workday:acme")
assert not ats.is_known_target("taleo:acme")
assert not ats.is_known_target("greenhouse:")
assert not ats.is_known_target("nohostnoseparator")
# a Teamtailor slug names a host: its own subdomain, or a whole domain
@@ -420,7 +421,7 @@ def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
assert ats.careers_url("teamtailor:acme") == "https://acme.teamtailor.com/jobs"
assert ats.careers_url("teamtailor:careers.acme.com") == "https://careers.acme.com/jobs"
assert ats.careers_url("workday:acme") == ""
assert ats.careers_url("taleo:acme") == ""
# --------------------------------------------------------------- body cap
@@ -570,7 +571,7 @@ def test_smartrecruiters_pages_by_offset_to_total_found(
urls.append(url)
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
@@ -588,7 +589,7 @@ def test_a_board_that_ends_short_of_its_total_is_incomplete(
def fake_get(client, url, host, slug, *, method="GET", payload=None):
return empty if "offset=2" in url else _page("smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
@@ -604,7 +605,7 @@ def test_a_repeated_row_does_not_count_twice_toward_the_total(
page["content"] = page["content"][1:]
return json.dumps(page).encode()
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
+761
View File
@@ -0,0 +1,761 @@
"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for
every host: the registry and its contract check, the fetch a host is handed,
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
a fake host stands in where a behaviour belongs to no real host yet."""
from __future__ import annotations
import json
import os
import subprocess
import sys
import types
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats, atshosts # noqa: E402
from plugins.atshosts import lever # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
# --------------------------------------------------------------- registry
def test_the_registry_finds_every_module() -> None:
names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py")
if not p.stem.startswith("_"))
assert sorted(atshosts.HOSTS) == names
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio", "teamtailor"}
assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items())
# The Sources page's select and the plugin's description follow the modules.
assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS
assert "Teamtailor" in ats.ATSSource.description
def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str,
modules: dict[str, str]) -> list[str]:
package = tmp_path / name
package.mkdir()
(package / "__init__.py").write_text("")
for module, source in modules.items():
(package / f"{module}.py").write_text(source)
monkeypatch.syspath_prepend(str(tmp_path))
return [str(package)]
_GOOD = """
NAME = "{name}"
LABEL = "Good"
PATTERNS = (r"good\\.example/([a-z]+)",)
def slug_ok(slug): return True
def allowed(slug): return lambda h: h == "good.example"
def careers_url(slug): return "https://good.example/" + slug
def feed_url(slug): return "https://good.example/" + slug + ".json"
def parse(body, slug): return []
"""
def test_the_registry_loads_ats_by_path_without_the_repo_root_on_the_path(
tmp_path: Path) -> None:
"""As `desk.sources.registry` does in a process the app did not set up:
another working directory, and `plugins` not importable at first."""
code = (
"import importlib.util, sys\n"
"assert importlib.util.find_spec('plugins') is None, 'plugins is importable'\n"
"from desk.sources import registry\n"
f"print(sorted(registry.available([{str(ROOT / 'plugins')!r}])))\n")
env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"}
done = subprocess.run([sys.executable, "-c", code], cwd=tmp_path, env=env,
capture_output=True, text=True, timeout=60, check=False)
assert done.returncode == 0, done.stderr
assert "'ats'" in done.stdout and "'search'" in done.stdout
def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
path = _package(tmp_path, monkeypatch, "hostsok", {
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
"_helpers": "X = 1\n"})
assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"]
def test_the_registry_orders_by_rank_then_name(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
path = _package(tmp_path, monkeypatch, "hostsranked", {
"alpha": _GOOD.format(name="alpha"), # the default, 100
"beta": _GOOD.format(name="beta") + "RANK = 5\n",
"gamma": _GOOD.format(name="gamma") + "RANK = 100\n"})
assert list(atshosts.load(path, "hostsranked")) == ["beta", "alpha", "gamma"]
def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
"""The Sources page's select opens on Greenhouse, which its hint names,
and a guessed slug on two hosts goes to the one probed first, as before."""
first = ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
"personio", "teamtailor"]
assert list(ats.HOSTS)[:8] == first
assert ats.ATSSource.description.startswith(
"Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, "
"Recruitee, Personio, Teamtailor")
assert all(module.RANK < atshosts.DEFAULT_RANK
for name, module in atshosts.HOSTS.items() if name in first)
def test_the_registry_refuses_a_module_missing_a_required_name(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
atshosts.load(path, "hostsbad")
def test_the_contract_check_names_each_problem() -> None:
module: Any = types.ModuleType("odd")
module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",)
module.slug_ok = module.allowed = module.careers_url = lambda slug: ""
module.page = lambda get, slug, cursor: None
module.probe_ok = lambda body: True
module.RANK, module.KEEP_CASE = "1", "yes"
found = atshosts.problems(module, "odd")
assert "NAME is 'other', not the module's name 'odd'" in found
assert "LABEL is not a non-empty string" in found
assert any(p.startswith("PATTERNS has '('") for p in found)
assert "page without rows" in found
assert "probe_ok without probe_url" in found
assert "RANK is not an int" in found and "KEEP_CASE is not a bool" in found
@pytest.mark.parametrize("host", ["greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio", "teamtailor"])
def test_a_slug_is_matched_whole(host: str) -> None:
assert ats.is_known_target(f"{host}:acme")
for slug in ("acme\n", "acme\r\n", " acme", "acme ", "ac me", "acme/x", "acme?x", "acme#x"):
assert not ats.is_known_target(f"{host}:{slug}"), slug
def test_a_slug_in_the_host_name_is_allowed_whatever_its_case() -> None:
for host, api in (("recruitee", "acme.recruitee.com"), ("personio", "acme.jobs.personio.de")):
allow = ats._allowed_host(host, "Acme")
assert allow(api) and allow(api.upper())
assert not allow(api.replace("acme", "other"))
# ------------------------------------------------------------ the fetch
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
return httpx.Client(transport=httpx.MockTransport(handler))
def test_get_sends_extra_headers_on_top_of_its_own() -> None:
seen: list[httpx.Headers] = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request.headers)
return httpx.Response(200, content=b"ok")
with _client(handler) as client:
get = ats._getter(client, "lever", "acme")
assert get("https://api.lever.co/v0/postings/acme",
headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok"
get("https://api.lever.co/v0/postings/acme")
assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme"
assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT
assert seen[1]["accept"] == "application/json, text/xml"
assert "x-board" not in seen[1]
@pytest.mark.parametrize("site", [
"metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa",
"app.localhost", "foo.0x7f000001", "printer.local.",
])
def test_get_refuses_a_local_name_before_any_request(site: str) -> None:
"""A Teamtailor slug may be any host name, so the allow-list admits
these; `_get` refuses them before a request is made."""
asked: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
asked.append(str(request.url))
return httpx.Response(200, content=b"<rss><channel/></rss>")
with _client(handler) as client:
# A trailing dot is no Teamtailor slug at all; the rest are, and are refused.
with pytest.raises(ValueError if site.endswith(".") else ats._Refused):
ats._fetch_rows(client, f"teamtailor:{site}")
for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/",
"https://2130706433/", "https://127.1/", "https://localhost/"):
with pytest.raises(ats._Refused):
ats._get(client, url, "teamtailor", site)
assert asked == []
@pytest.mark.parametrize("location", [
"https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x",
"https://localhost/x",
])
def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(302, headers={"location": location})
with _client(handler) as client, pytest.raises(ats._Refused):
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
def test_get_takes_port_443_written_out_and_refuses_another() -> None:
with _client(lambda request: httpx.Response(200, content=b"[]")) as client:
assert ats._get(client, "https://api.lever.co:443/v0/postings/acme",
"lever", "acme") == b"[]"
with pytest.raises(ats._Refused):
ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme")
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
"""Bound before the replacement, and still reaching it: `_get` is looked
up when a request is made."""
client = object()
get = ats._getter(client, "lever", "acme") # type: ignore[arg-type]
calls: list[tuple[Any, ...]] = []
def replaced(*args: Any, **kwargs: Any) -> bytes:
calls.append((args, kwargs))
return b""
monkeypatch.setattr(ats, "_get", replaced)
get("https://api.lever.co/v0/postings/acme", headers={"X": "1"})
get("https://api.lever.co/v0/postings/acme", method="POST", payload={})
assert calls == [
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
{"method": "GET", "headers": {"X": "1"}}),
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
{"method": "POST", "payload": {}})]
# ------------------------------------------------------- a fake host
def _fake_host(*, detail: Callable[..., Any] | None = None,
robots: bool = False) -> types.ModuleType:
"""A host `fake` whose feed is a JSON list of ids, each role with a
teaser for a description, and, with `detail`, a per-role page; with
`robots`, its listing is the page robots.txt must allow."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
"description": "teaser", "remote": None, "salary_text": ""}
for i in json.loads(body)]
if detail is not None:
module.detail = detail
if robots:
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
return module
def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError:
request = httpx.Request("GET", url)
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
@pytest.fixture
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
"""`ats._get` answering from a dict of URL -> body or exception, and
remembering what it was asked. No delay between requests."""
answers: dict[str, Any] = {}
asked: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
www_twin=False):
asked.append(url)
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
raise ats._Refused(url)
answer = answers.get(url, _status(404, url))
if isinstance(answer, BaseException):
raise answer
return answer
monkeypatch.setattr(ats, "_get", fake_get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
return {"answers": answers, "asked": asked}
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
state: dict[str, str] | None = None) -> tuple[list[Any], Any]:
monkeypatch.setitem(atshosts.HOSTS, "fake", module)
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
if state is not None:
source.state = state
return list(source.fetch()), source.state
def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
"""The fake host's detail: the role's own page is its description."""
return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(),
"location": "Copenhagen"}
# ---------------------------------------------------------------- detail
def test_detail_is_read_for_unread_roles_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
module = _fake_host(detail=_detail)
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
# What detail returns replaces the listing's field; a role not read keeps the listing's.
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert state["ats:complete:fake:acme"] == "1"
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
ids = [str(n) for n in range(5)]
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
module = _fake_host(detail=_detail)
items, state = _run(monkeypatch, module)
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
assert state["ats:complete:fake:acme"] == "1"
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": _status(429), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
assert state["ats:complete:fake:acme"] == "1"
def test_ids_no_longer_listed_leave_the_detailed_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""A listing cut short says nothing about the roles it left out, so their
ids stay, and the next pull does not read them in detail again."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False))
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
assert state["ats:complete:fake:acme"] == ""
def test_fetch_board_reads_the_listing_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
rows, complete = ats.fetch_board("fake:acme")
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
assert web["asked"] == [f"{base}/list"]
def test_a_host_without_detail_keeps_the_listings_description(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]'
items, state = _run(monkeypatch, _fake_host())
assert [i.body for i in items] == ["teaser"]
assert "ats:detailed:fake:acme" not in state
def test_a_detail_answering_404_or_410_counts_as_read(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
f"{base}/3": b"three"}) # 1 answers 404
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [i.body for i in items] == ["", "", "three"]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
"""Two roles that always fail and a cap of two: the third is read on the
second pull, since roles never tried go before those that failed."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
module = _fake_host(detail=_detail)
_items, state = _run(monkeypatch, module)
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
assert [i.body for i in items] == ["", "", "three", "four"]
# With nothing new left, the failed ones are tried again, within the cap.
web["asked"].clear()
web["answers"][f"{base}/1"] = b"one at last"
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one at last", "", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
assert json.loads(state["ats:detailed:fake:acme"]) == []
def test_failed_ids_no_longer_listed_leave_the_failed_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_detail_fills_only_the_fields_a_later_pull_keeps(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any],
caplog: pytest.LogCaptureFixture) -> None:
"""`salary_text`, `department` and `title` would be rewritten from the
listing on the next pull, so detail does not fill them."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]'})
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
"salary_min": 60000, "salary_max": 70000,
"posted": datetime(2026, 9, 1, tzinfo=UTC),
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
"title": "Another title"}
caplog.set_level("DEBUG", logger=ats.log.name)
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
assert (item.body, item.title, item.published_at) == (
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
assert "department, salary_text, title left out" in caplog.text
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
"salary_min", "salary_max", "posted"}
def _paged_host() -> types.ModuleType:
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
module: Any = _fake_host(detail=_detail)
del module.feed_url, module.parse
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
index = int(cursor or 0)
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
module.page = page
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
assert not atshosts.problems(module, "fake")
return module
def test_a_paged_host_reads_every_page_then_each_role_once(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
f"{base}/1": b"one", f"{base}/2": b"two"})
items, state = _run(monkeypatch, _paged_host())
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "two"]
assert state["ats:complete:fake:acme"] == "1"
web["asked"].clear()
items, state = _run(monkeypatch, _paged_host(), state)
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
assert [i.body for i in items] == ["", ""]
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
f"{base}/1": b"one"})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
_items, state = _run(monkeypatch, _paged_host(), state)
assert state["ats:complete:fake:acme"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The consumer stops after the first role, so the second, read but not
stored, is read again on the next pull."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
roles: Any = source.fetch() # a generator, which `close` stops
assert next(roles).body == "one"
roles.close()
assert "ats:detailed:fake:acme" not in source.state
# ------------------------------------------------------------ robots.txt
ROBOTS = "https://jobs.fake.example/robots.txt"
def test_robots_that_allow_the_page_let_the_board_be_read(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
@pytest.mark.parametrize("robots", [
b"User-agent: *\nDisallow: /acme/\n",
# A rule for this app by name counts as well as the one for everyone.
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
.encode(),
])
def test_robots_that_disallow_the_page_fail_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert items == []
assert web["asked"] == [ROBOTS] # the listing is never fetched
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
assert "ats:ok:fake:acme" not in state
@pytest.mark.parametrize(("robots", "page", "allowed"), [
# The longest matching rule wins, whatever the order of the lines.
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
# An allow wins a tie.
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
# `*` inside a path, and the query counts as part of it.
("User-agent: *\nDisallow: /*company\n",
"https://career5.successfactors.eu/career?company=x", False),
# `$` anchors the end.
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
# An empty Disallow allows.
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
# A byte order mark and CRLF line ends.
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
# Paths are case-sensitive.
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
# A group for this app by name counts, a group for someone else does
# not, and a token matches whole, not as a substring.
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
"https://x.example/jobs", False),
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
# Two agents share one group, and a comment is no rule.
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
"https://x.example/jobs/1", False),
# A group for this app that allows does not undo the `*` group's refusal.
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
"https://x.example/jobs", False),
# Rules before any user-agent line belong to no group.
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
])
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
assert ats._robots_allows(robots.encode(), page) is allowed
def test_a_robots_verdict_is_kept_for_a_day(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
module = _fake_host(robots=True)
_items, state = _run(monkeypatch, module)
assert web["asked"] == [ROBOTS]
# Within the day the kept verdict holds, even though the site changed.
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert (items, web["asked"]) == ([], [])
# A day on, robots.txt is read again.
kept = json.loads(state["ats:robots:fake:acme"])
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
state["ats:robots:fake:acme"] = json.dumps(kept)
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
assert len(items) == 1 and "ats:failing:fake:acme" not in state
@pytest.mark.parametrize("code", [404, 410, 401, 403])
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
@pytest.mark.parametrize("failure", [
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
])
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert items == [] and web["asked"] == [ROBOTS]
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
"robots.txt could not be read"
assert "ats:robots:fake:acme" not in state
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
where a robots.txt that disallows everything is served."""
def handler(request: httpx.Request) -> httpx.Response:
if str(request.url) == ROBOTS:
return httpx.Response(301, headers={"location": location})
if request.url.path == "/robots.txt":
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
return httpx.Response(200, text='["1"]')
return handler
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Disallowed):
ats._check_robots(client, "fake:acme", {})
@pytest.mark.parametrize("location", [
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
"http://www.jobs.fake.example/robots.txt",
])
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
ats._check_robots(client, "fake:acme", {})
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Refused):
ats._get(client, ROBOTS, "fake", "acme")
def test_fetch_board_honours_robots_txt(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with pytest.raises(ats._Disallowed) as caught:
ats.fetch_board("fake:acme")
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
# ----------------------------------------------------------------- Lever EU
def test_a_lever_slug_may_name_the_eu_instance() -> None:
assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom")
assert not ats.is_known_target("lever:us/acme")
assert not ats.is_known_target("lever:eu/")
assert not ats.is_known_target("lever:eu/a/b")
assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme"
assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom"
assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
eu = ats._allowed_host("lever", "eu/tomtom")
assert eu("api.eu.lever.co") and not eu("api.lever.co")
assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co")
def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None:
urls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET"):
urls.append(url)
assert ats._allowed_host(host, slug)(httpx.URL(url).host)
return (FIXTURES / "lever.json").read_bytes()
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type]
assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"]
assert rows and complete
def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"})
with _client(handler) as client, pytest.raises(ats._Refused):
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
+110 -10
View File
@@ -4,7 +4,11 @@ site's pages are read."""
from __future__ import annotations
import json
import re
import types
from collections.abc import Set
from pathlib import Path
from typing import Any
import httpx
import pytest
@@ -36,16 +40,59 @@ def test_boards_in_names_readable_hosts_with_their_slug() -> None:
def test_boards_in_keeps_hosts_the_app_cannot_read() -> None:
page = """
<a href="https://novo.wd3.myworkdayjobs.com/en-US/careers">Workday</a>
<a href="https://performancemanager.successfactors.eu/career?company=x">SF</a>
<a href="https://novo.taleo.net/careersection/2/jobsearch.ftl">Taleo</a>
<a href="https://acme.teamtailor.com/jobs">TT</a>
"""
assert boards_in(page) == [
("workday", "novo", "https://novo.wd3.myworkdayjobs.com"),
("successfactors", "", "performancemanager.successfactors.eu"),
("taleo", "", ".taleo.net"),
("teamtailor", "acme", "https://acme.teamtailor.com")]
def test_boards_in_names_a_lever_eu_board_by_its_region() -> None:
page = ('<a href="https://jobs.eu.lever.co/tomtom/abc-123">EU</a>'
'<script src="https://api.eu.lever.co/v0/postings/tomtom?mode=json"></script>'
'<a href="https://jobs.lever.co/whereby">US</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"),
("lever", "whereby")]
def test_boards_in_lowercases_a_slug_unless_the_host_keeps_its_case(
monkeypatch: pytest.MonkeyPatch) -> None:
"""A Workday-shaped host with two groups and `KEEP_CASE`: the site keeps
its case, and a site called `jobs` is a site, since only group 1 is
checked against the hosts' own path words."""
workday = re.compile(r"https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/"
r"(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.IGNORECASE)
monkeypatch.setattr(mod, "HOSTS", (("workday", workday),
*(h for h in mod.HOSTS if h[0] != "workday")))
monkeypatch.setattr(mod, "_KEEP_CASE", frozenset({"workday"}))
page = ('<a href="https://lego.wd103.myworkdayjobs.com/LEGO_External">a</a>'
'<a href="https://redhat.wd5.myworkdayjobs.com/en-US/jobs">b</a>'
'<a href="https://jobs.smartrecruiters.com/BoschGroup">c</a>'
'<a href="https://jobs.eu.lever.co/TomTom">d</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [
("workday", "lego.wd103/LEGO_External"), ("workday", "redhat.wd5/jobs"),
("smartrecruiters", "boschgroup"), ("lever", "eu/tomtom")]
# A host's own path word in group 1 still names no board.
assert boards_in('<a href="https://jobs.smartrecruiters.com/api">x</a>') == []
def test_no_module_of_the_eight_keeps_its_case() -> None:
"""SmartRecruiters and Ashby ignore case (checked live, 2026-09-29), so a
slug from a page stays lowercased as it always was."""
assert not mod._KEEP_CASE
def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
"""Wave by wave, a host named here becomes readable by its module alone."""
taleo: Any = types.ModuleType("taleo")
taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",)
hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE)
assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [
r"https?://([a-z0-9-]+)\.taleo\.net"]
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
@@ -81,7 +128,7 @@ class Site:
is a page that did not answer. Remembers the order of fetches."""
def __init__(self, pages: dict[str, tuple[str, str] | str],
boards: set[tuple[str, str]] = frozenset()) -> None:
boards: Set[tuple[str, str]] = frozenset()) -> None:
self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()}
self.boards = boards
self.fetched: list[str] = []
@@ -138,12 +185,12 @@ def test_a_careers_pages_own_link_is_followed_one_level_down() -> None:
def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None:
site = Site({
"https://acme.com/": "<p>welcome</p>",
"https://acme.com/careers": '<a href="https://acme.wd3.myworkdayjobs.com/x">Jobs</a>',
"https://acme.com/careers": '<a href="https://acme.taleo.net/careersection/x">Jobs</a>',
}, boards={("greenhouse", "acme")})
found = run(site)
assert found == Discovery(ats="workday", ats_slug="acme",
assert found == Discovery(ats="taleo", ats_slug="",
careers_url="https://acme.com/careers",
evidence="https://acme.wd3.myworkdayjobs.com")
evidence=".taleo.net")
assert not found.readable
# The page named the host: a probe could only find another tenant's
# board of the same name, like the greenhouse:acme this fake web has.
@@ -223,6 +270,59 @@ def test_slug_probes_find_a_board_the_pages_never_link() -> None:
assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme"
@pytest.mark.parametrize(("boards", "found"), [
({("greenhouse", "acme"), ("ashby", "acme")}, "greenhouse:acme"),
({("lever", "acme"), ("ashby", "acme")}, "lever:acme"),
({("smartrecruiters", "acme"), ("personio", "acme")}, "smartrecruiters:acme"),
({("lever", "eu/acme"), ("personio", "acme")}, "lever:eu/acme"),
])
def test_a_slug_on_two_hosts_goes_to_the_one_probed_first(
boards: set[tuple[str, str]], found: str) -> None:
site = Site({}, boards=boards)
assert run(site).board == found
def test_the_probe_order_is_the_one_before_modules() -> None:
assert list(mod._PROBES)[:7] == ["greenhouse", "lever", "ashby", "smartrecruiters",
"recruitee", "personio", "teamtailor"]
site = Site({})
run(site)
assert site.probed[:8] == [("greenhouse", "acme"), ("lever", "acme"), ("lever", "eu/acme"),
("ashby", "acme"), ("smartrecruiters", "acme"),
("recruitee", "acme"), ("personio", "acme"),
("teamtailor", "acme")]
def test_a_lever_eu_board_is_found_by_its_probe() -> None:
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>"},
boards={("lever", "eu/acme")})
found = run(site)
assert found.board == "lever:eu/acme"
assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json"
assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme"))
def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None:
asked: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
asked.append(str(request.url))
if request.url.host == "api.eu.lever.co":
return httpx.Response(200, json=[])
if request.url.host == "api.ashbyhq.com":
return httpx.Response(200, content=b'{"jobs":[]}')
return httpx.Response(404)
with httpx.Client(transport=httpx.MockTransport(handler)) as client:
assert mod.probe(client, "lever", "eu/acme")
assert not mod.probe(client, "lever", "acme")
assert not mod.probe(client, "ashby", "acme") # an empty listing is no board
assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json",
"https://api.lever.co/v0/postings/acme?limit=1&mode=json",
"https://api.ashbyhq.com/posting-api/job-board/acme"]
assert "workable" not in mod._PROBES and "lever" in mod._PROBES
def test_a_site_that_never_answers_gives_an_empty_discovery() -> None:
site = Site({})
assert run(site) == Discovery()
@@ -251,7 +351,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa
{"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen",
"board": "ashby:pleo"},
{"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK",
"city": "Bagsværd", "ats": "successfactors",
"city": "Bagsværd", "ats": "taleo",
"careers_url": "https://www.novonordisk.com/careers.html"},
{"name": "Odd", "board": "nosuch:odd"},
]}))
@@ -266,7 +366,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa
assert (sub.target, sub.company) == ("ashby:pleo", "Pleo")
novo = store.company("novo-nordisk")
assert novo is not None
assert (novo.ats, novo.careers_url) == ("successfactors",
assert (novo.ats, novo.careers_url) == ("taleo",
"https://www.novonordisk.com/careers.html")
odd = store.company("odd")
assert odd is not None and odd.ats == ""
+18
View File
@@ -546,6 +546,24 @@ def test_replacing_or_detaching_a_board_forgets_that_it_was_failing(
assert failing_boards(store) == {}
def test_replacing_or_detaching_a_board_forgets_its_robots_and_detail_records(
store: Store, boards: Boards) -> None:
kept = ("ats:robots:", "ats:detailed:", "ats:detailfailed:")
sources.set_board(store, "acme", "greenhouse:acme", name="Acme")
for key in kept:
store.state[key + "greenhouse:acme"] = "[]"
store.state["ats:robots:lever:other"] = "{}"
sources.set_board(store, "acme", "ashby:acme")
assert not any(key + "greenhouse:acme" in store.state for key in kept)
assert "ats:robots:lever:other" in store.state
for key in kept:
store.state[key + "ashby:acme"] = "[]"
sources.detach_board(store, "acme")
assert not any(key + "ashby:acme" in store.state for key in kept)
def test_search_params_keeps_only_what_the_provider_reads() -> None:
assert sources.search_params("jobindex", " python ", area=1, region="CaRE_1nn_cSU") == {
"q": "python", "area": 1}