diff --git a/app/jobs/discover.py b/app/jobs/discover.py index 2e35765..b52b8bb 100644 --- a/app/jobs/discover.py +++ b/app/jobs/discover.py @@ -5,10 +5,12 @@ The seed list and the LinkedIn flow both start from a company name and a website, not from a board. This module gets from the one to the other: it fetches the site's careers page, or the pages the home page links to as careers, and looks for a link into one of the hosts in `HOSTS`. A readable -host (one the `ats` plugin parses) becomes a `board` target the pull can -read; any other host is recorded as `ats`, with its slug where the URL -carries one, so the company is kept as one the app cannot read yet rather -than dropped. +host (one with a module in `plugins/atshosts/`, which the `ats` plugin +parses) becomes a `board` target the pull can read; any other host is +recorded as `ats`, with its slug where the URL carries one, so the company is +kept as one the app cannot read yet rather than dropped. The patterns and +probes of a readable host are its module's; the hosts only named are in +`_UNREADABLE` here, and one of them that gains a module is read from then on. A Teamtailor site on the company's own domain names no slug: the page only loads Teamtailor's CDN. The board there is the site's own host, and it is @@ -16,7 +18,8 @@ taken only once that host answers `/jobs.rss` with a Teamtailor feed. Only when no page names a host at all are the hosts with a cheap per-slug endpoint (`_PROBES`) probed with the company's own slug, because a site that -draws its jobs from an iframe or a script leaves no link to find. A page that +draws its jobs from an iframe or a script leaves no link to find. Lever is +probed on its EU instance as well, as `lever:eu/`. A page that names an unreadable host is the answer: a probe on a common name can hit another tenant's board. @@ -37,13 +40,15 @@ from __future__ import annotations import logging import re -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, replace +from types import ModuleType from urllib.parse import urljoin import httpx from desk.sources import USER_AGENT +from plugins import atshosts log = logging.getLogger(__name__) @@ -53,55 +58,61 @@ MAX_BYTES = 2 << 20 MAX_PAGES = 6 #: The hosts the `ats` plugin parses; a match on one of these is a `board`. -READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters", - "recruitee", "personio", "teamtailor") +READABLE: tuple[str, ...] = tuple(atshosts.HOSTS) -#: host -> pattern over a URL. Group 1, where there is one, is the slug the -#: host names the company by; a host with no group is recognised but its -#: board has no slug the app could use. Order matters only for readability: -#: the readable hosts come first so the first match on a page is the one the -#: app can read. -HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple( - (host, re.compile(pattern, re.IGNORECASE)) for host, pattern in ( - ("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"), - ("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"), - ("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"), - ("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"), - ("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"), - ("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"), - ("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"), - ("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"), - ("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"), - ("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"), - ("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"), - ("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"), - # No slug: a career site on the company's own domain, which only loads - # Teamtailor's CDN. `_discover` reads its board off the page's host. - ("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"), - # Not readable. The slug, where the URL has one, is kept for the day - # a parser exists. - ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"), - ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"), - ("icims", r"https?://([a-z0-9-]+)\.icims\.com"), - ("taleo", r"\.taleo\.net"), - ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"), - ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"), - ("join", r"join\.com/companies/([a-z0-9-]+)"), - ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"), - ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"), - ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"), - ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"), - ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"), - ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"), - ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"), - ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"), - ("eightfold", r"\.eightfold\.ai"), - ("phenom", r"phenompeople\.com|phenom\.com/careers"), - ("avature", r"\.avature\.net"), - ("oracle", r"oraclecloud\.com/hcmUI"), - ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), - ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"), - )) +#: host -> pattern over a URL, for the hosts discovery names but the app +#: cannot read. Group 1, where there is one, is the slug the host names the +#: company by, kept for the day a parser exists; a host with no group is +#: recognised but its board has no slug the app could use. +_UNREADABLE: tuple[tuple[str, str], ...] = ( + ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"), + ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"), + ("icims", r"https?://([a-z0-9-]+)\.icims\.com"), + ("taleo", r"\.taleo\.net"), + ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"), + ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"), + ("join", r"join\.com/companies/([a-z0-9-]+)"), + ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"), + ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"), + ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"), + ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"), + ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"), + ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"), + ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"), + ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"), + ("eightfold", r"\.eightfold\.ai"), + ("phenom", r"phenompeople\.com|phenom\.com/careers"), + ("avature", r"\.avature\.net"), + ("oracle", r"oraclecloud\.com/hcmUI"), + ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), + ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"), +) + + +def _hosts(modules: Mapping[str, ModuleType], + unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]: + """Every host's patterns, compiled: the readable hosts' from their + modules first, so the first match at one place on a page is the one the + app can read, then the unreadable ones. An unreadable host that has + gained a module is left out here, since its module's patterns replace + it.""" + out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS] + out += [(host, pattern) for host, pattern in unreadable if host not in modules] + return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out) + + +#: host -> pattern over a URL, every host discovery can name. The slug is +#: group 1, or, for a pattern with more groups, the groups that matched +#: joined by "/" (Lever's `eu/`). +HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE) + +#: The hosts whose slugs keep the case the URL wrote them in, by their +#: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its +#: listing and its robots.txt paths are case-sensitive. Every other slug is +#: lowercased, which is how a page that writes `BoschGroup` and one that +#: writes `boschgroup` name one board. +_KEEP_CASE = frozenset(name for name, module in atshosts.HOSTS.items() + if getattr(module, "KEEP_CASE", False)) #: What a Teamtailor feed carries and a page that only looks like one does not: #: the namespace of `tt:locations`, declared on `` even when no role is open. @@ -125,7 +136,7 @@ _ANCHOR = re.compile(r"]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?) re.IGNORECASE | re.DOTALL) _TAGS = re.compile(r"<[^>]+>") -#: Words a pattern's slug group can catch that name no board: the hosts' +#: Words a pattern's group 1 can catch that name no board: the hosts' #: own path segments, the analytics subdomain Recruitee sites embed, and #: the CDN and the admin app Teamtailor sites load from (cdn.teamtailor.com, #: app.teamtailor.com). @@ -161,9 +172,13 @@ def boards_in(text: str) -> list[Found]: hits: list[tuple[int, str, str, str]] = [] for host, pattern in HOSTS: for m in pattern.finditer(text): - slug = (m.group(1) if m.groups() and m.group(1) else "").lower() - if slug in _NOT_SLUGS: + # Only group 1 names a board or a host's own path word; a later + # group may be anything, a Workday site called `jobs` included. + if m.re.groups and (m.group(1) or "").lower() in _NOT_SLUGS: continue + slug = "/".join(part for part in m.groups() if part) + if host not in _KEEP_CASE: + slug = slug.lower() hits.append((m.start(), host, slug, m.group(0))) for _pos, host, slug, url in sorted(hits): if (host, slug) in seen: @@ -237,25 +252,28 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]: return "", "" -#: host -> a cheap URL that answers 2xx only when the slug is a board there. -#: Workable is left out: its API sits behind a rate limit that a run of -#: guesses trips, and a Workable site links its board plainly anyway. -_PROBES = { - "greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}", - "lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json", - "ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}", - "smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1", - "recruitee": "https://{slug}.recruitee.com/api/offers/", - "personio": "https://{slug}.jobs.personio.de/xml", - "teamtailor": "https://{slug}.teamtailor.com/jobs.rss", -} +#: host -> a cheap URL that answers 2xx only when the slug is a board there, +#: from each module's `probe_url`. Workable has none: its API sits behind a +#: rate limit that a run of guesses trips, and a Workable site links its +#: board plainly anyway. +_PROBES: dict[str, Callable[[str], str]] = { + name: module.probe_url for name, module in atshosts.HOSTS.items() + if hasattr(module, "probe_url")} + + +def _probe_slugs(host: str, guess: str) -> tuple[str, ...]: + """The slugs a guessed name is probed as on `host`: the name itself, and + for Lever the name on its EU instance too (the module's `probe_slugs`).""" + module = atshosts.HOSTS[host] + return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,) def probe(client: httpx.Client, host: str, slug: str) -> bool: """Whether `host` has a board called `slug`, by its cheapest endpoint. - Ashby and SmartRecruiters answer an empty listing for an unknown slug, - so there a listing without jobs counts as no board.""" - url = _PROBES[host].format(slug=slug) + A host that answers an unknown slug with an empty listing (Ashby, + SmartRecruiters, ...) says in its module's `probe_ok` what a listing + with a board looks like.""" + url = _PROBES[host](slug) try: response = client.get(url, timeout=TIMEOUT, follow_redirects=False, headers={"User-Agent": USER_AGENT}) @@ -263,18 +281,8 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool: return False if not response.is_success: return False - body = response.content - if host == "ashby": - return b'"jobs":[' in body and b'"jobs":[]' not in body - if host == "smartrecruiters": - return b'"totalFound":' in body and b'"totalFound":0' not in body - if host == "recruitee": - return b'"offers":[' in body and b'"offers":[]' not in body - if host == "personio": - return b"" in body - if host == "teamtailor": - return b"" in body - return True + module = atshosts.HOSTS[host] + return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True Fetch = Callable[[httpx.Client, str], tuple[str, str]] @@ -377,9 +385,10 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch, # probe could only add another tenant's board of the same name. if unreadable is not None: return unreadable - for slug in slug_guesses(name, site): + for guess in slug_guesses(name, site): for host in _PROBES: - if probe_slug(client, host, slug): - return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, - careers_url=careers_page, evidence=_PROBES[host].format(slug=slug)) + for slug in _probe_slugs(host, guess): + if probe_slug(client, host, slug): + return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, + careers_url=careers_page, evidence=_PROBES[host](slug)) return Discovery(careers_url=careers_page) diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 3eab0bb..cb73677 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -111,8 +111,12 @@ def detach_board(store: Store, key: str) -> int: def _forget_failing(store: Store, target: str) -> None: """Drop what the pull recorded about a board nobody reads any more: a record of `target` failing would otherwise outlive the board, and count - in `app companies check` for good.""" + in `app companies check` for good. The `ats` plugin's robots.txt verdict + and its lists of roles read and failed in detail go with it, since they + are about that board alone and would only grow stale.""" store.state.pop(FAILING + target, None) + for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"): + store.state.pop(key + target, None) def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]: diff --git a/app/models.py b/app/models.py index 119adfc..1e83ccd 100644 --- a/app/models.py +++ b/app/models.py @@ -661,7 +661,7 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason) #: read as when a role was posted, checked against live boards on #: 2026-09-29: #: -#: - `greenhouse`: `plugins.ats._parse_greenhouse` already prefers +#: - `greenhouse`: `plugins.atshosts.greenhouse.parse` already prefers #: `first_published` over `updated_at`. A live board (adyen) showed a job #: with `first_published` five months before `updated_at`, proving the #: first stays fixed while the second moves on an edit — the resettable diff --git a/docs/plan-hosts-and-by-hand.md b/docs/plan-hosts-and-by-hand.md index fd478f6..24df3af 100644 --- a/docs/plan-hosts-and-by-hand.md +++ b/docs/plan-hosts-and-by-hand.md @@ -46,32 +46,52 @@ The 49 were missed because the slug is not the company's name (`BoschGroup`, `ji ## Contracts between packages -Host module, `plugins/atshosts/.py`: +Host module, `plugins/atshosts/.py`, as built by WP0. The docstring of `plugins/atshosts/__init__.py` holds the same contract and is the one to keep current. A module imports `desk` and `._shared` only, never `plugins.ats`. ```python -NAME = "workday" # the host of a target `workday:` -LABEL = "Workday" # the Sources page's select -PATTERNS = (r"https?://...",) # discover: over a URL, group 1 is the slug +NAME = "workday" # the host of a target `workday:`; equals the module's file name +LABEL = "Workday" # the Sources page's select +PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE +RANK = 100 # optional int, default 100; lower comes first +KEEP_CASE = True # optional, default False; keep a discovered slug's case -def slug_ok(slug: str) -> bool: ... # strict; a slug that fails is no target -def allowed(slug: str) -> Callable[[str], bool]: ... # hostnames a fetch may reach -def careers_url(slug: str) -> str: ... # the page a person opens -def robots_page(slug: str) -> str: ... # optional; the URL robots.txt must allow +def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch +def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach +def careers_url(slug: str) -> str: ... # the page a person opens # one of the two -def feed_url(slug: str) -> str: ... +def feed_url(slug: str) -> str: ... # the whole board in one answer def parse(body: bytes, slug: str) -> list[Row]: ... -def page(get: Get, slug: str, cursor: Any) -> Page: ... +def page(get: Get, slug: str, cursor: Any) -> Page: ... # a paged board, one page a call def rows(decoded: Any, slug: str) -> list[Row]: ... # optional -def detail(get: Get, slug: str, row: Row) -> Row: ... -def probe_url(slug: str) -> str: ... -def probe_ok(body: bytes) -> bool: ... +def fallback_url(slug: str) -> str: ... # feed hosts: a smaller feed, read once when feed_url is over the byte cap +def robots_page(slug: str) -> str: ... # the URL robots.txt must allow; on a host allowed(slug) admits +def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: ... # fields from the role's own page +def probe_url(slug: str) -> str: ... # discovery: answers 2xx only for a board +def probe_ok(body: bytes) -> bool: ... # discovery: when 2xx is not enough +def probe_slugs(guess: str) -> tuple[str, ...]: ... # the slugs a guessed name is probed as; default (guess,) ``` -`Get` is `get(url, *, method="GET", payload=None, headers=None) -> bytes`. `Row` and `Page` are as in `plugins/ats.py` today. `plugins/atshosts/__init__.py` lists the modules with `pkgutil.iter_modules`, sorted by name, so a new host registers by existing. +`Get`, `Row` and `Page` are in `plugins/atshosts/_shared.py`, with the helpers several hosts share: `only(host)` for `allowed`, `path_slug_ok` and `label_slug_ok` for `slug_ok`, `parse_dt`, `is_web_url`, `num`, `yearly_amount` and `pay_line`. `plugins/atshosts/__init__.py` lists the modules with `pkgutil.iter_modules`, checks each against the contract at import (an ImportError names the module and what it lacks), and orders `HOSTS` by `(RANK, NAME)`. The eight first hosts hold ranks 10 to 80, which keeps the order they had before the modules: the select opens on Greenhouse, and a guessed slug on two hosts goes to the one probed first. + +Order and names. `RANK` orders the Sources page's select, the plugin's description and discovery's probes. `NAME` equals the file name. A host that `discover.py` names in its own `_UNREADABLE` table is taken from its module as soon as the module exists, so a wave 1 package does not edit `discover.py`. + +Patterns. The slug is group 1, or, when a pattern has more groups, the groups that matched joined by "/" (`https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)` gives `lego.wd103/LEGO_External`). Only group 1 is checked against discovery's words that name no board (`api`, `jobs`, `embed`, ...). The slug is lowercased unless the module sets `KEEP_CASE = True`, which a host with case-sensitive paths needs, since robots.txt paths are case-sensitive too. A pattern without a group names the host with no slug. + +Slugs. `slug_ok` uses `re.fullmatch` (a `$` lets a trailing newline through) and refuses anything that could add a path, query, fragment, port or user to the URLs built from it. A slug of several parts joins them with "/": Lever's EU boards are `lever:eu/`. + +`get(url, *, method="GET", payload=None, headers=None) -> bytes` is `ats._get` bound to the board. A POST sends `payload` as JSON; `headers` are added to the default User-Agent and Accept and replace either where they name it. It fetches https on port 443 only, never an IP address or a local name (`localhost`, `.local`, `.internal`, `.lan`, `.home.arpa`), follows at most three redirects each checked against `allowed(slug)`, cuts the body at `MAX_BYTES` and ends within `DEADLINE`. It returns the body and raises `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError` on a timeout or a connection failure, and another exception, `ValueError` among them, for a refused host, an oversized body or a broken redirect. A module lets these through. + +`Page` is `(decoded, results, total, cursor)`: the decoded page handed to `rows`, its raw results as dicts with an `id` (counted by distinct id for completeness), the total the host states or None, and the next cursor or None after the last page. The first call gets cursor None. A board with a total is complete when as many distinct ids were read, one without when the read ended on an empty page; a 429 after the first page ends the read, incomplete. A paged module may also have `parse(body, slug)` for one page's body, for its tests. + +A `Row` has `source_id`, `title`, `location`, `url`, `posted` (aware datetime or None), `department`, `description` (plain text), `remote` (True, False, a word or None) and `salary_text`, and, where the host states pay, `salary_min`, `salary_max` (yearly integers) and `currency`. + +`detail` returns a dict of what the role's own page states. The shared code calls it for roles not yet read (`ats:detailed:`), at most `DETAIL_CAP` (200) per board per pull, `PAGE_DELAY` apart, the roles never tried before those whose detail failed (`ats:detailfailed:`). Only `DETAIL_FIELDS` are taken: `description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the listing's empty value (`upsert_role`'s keep-filled columns, the salary figures, and `posted_at`, which keeps the earliest date). Any other key is dropped with a debug log line: `title`, `url`, `department` and `salary_text` cannot be filled from `detail`, since the listing rewrites them every pull. A role not read in detail on a pull is sent with an empty description. A 404 or 410 counts as read, any other failure is retried on a later pull, and a 429 ends the detail reads for that board and pull. Completeness is decided by the listing alone. + +robots.txt. For a module with `robots_page`, the shared code reads `https:///robots.txt` before the listing, at most once a day per board (`ats:robots:`), following a redirect to the host's `www.` twin or back and no other. It fails the board with "robots.txt disallows it" when the rules for `*` or for the app's own token disallow the page, matched per RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a final `$` in paths, paths case-sensitive, an empty Disallow allows. A 4xx other than 429 means there are no rules, which allows. A 5xx, a 429, a timeout, a connection error or a redirect elsewhere fails the board with "robots.txt could not be read", and the next pull asks again. Slugs: Workday `lego.wd103/LEGO_External`; SuccessFactors, Phenom and the sitemap reader take the site's hostname, with a path where the site has one (`jobs.teradyne.com/MiR`); Oracle `/`. diff --git a/plugins/__init__.py b/plugins/__init__.py index 5433535..94ff7ff 100644 --- a/plugins/__init__.py +++ b/plugins/__init__.py @@ -1,8 +1,10 @@ """Drop-in plugins, discovered by `plugin_paths` in `config.yaml`. -A plugin file imports `desk` only — see `tests/test_architecture.py`. This -`__init__.py` makes the directory importable as `plugins.` for the rest -of roledesk (`jobs/pull.py`, `cli.py`, the tests), which the plugin contract -itself never relies on: `desk.sources.registry` finds `ats.py` by path, not -by this package. +A plugin file imports `desk` only — see `tests/test_architecture.py` — and +its own helpers in this directory. This `__init__.py` makes the directory +importable as `plugins.`, for the rest of roledesk (`jobs/pull.py`, +`cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the +package `plugins/atshosts/`. `desk.sources.registry` still finds `ats.py` by +path, not by this package; `ats.py` puts the repo root on `sys.path` itself +when the app has not, so that `plugins.atshosts` imports either way. """ diff --git a/plugins/ats.py b/plugins/ats.py index ba6b618..14bfd7e 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -1,69 +1,83 @@ -"""ATS boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, -Teamtailor. +"""ATS boards: a company's own careers page on one of the hosts in +`plugins/atshosts/`, one module per host. -Every one of these hosts is a company's own careers page speaking public, -unauthenticated JSON (Personio, Teamtailor: XML) — first-party, no scraping, and a role -that drops off the board is *closed*, which is the cheapest staleness check -there is. +Every one of these hosts serves a company's own list of roles as public, +unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that +drops off the board is *closed*, which is the cheapest staleness check there +is. What a host says and how its feed reads is its module's; this file holds +what every host shares: the fetch and its allow-list, paging and +completeness, the per-role detail reads, robots.txt, the classifier, and the +plugin itself. -The field map and the redirect allow-list are ported from -`openings.go` and `cmd/seedgen/main.go` in +The allow-list and the classifier are ported from `openings.go` and +`cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including -`classify`'s nine functions and their keywords. Workable, SmartRecruiters, -Recruitee and Personio are new: jobhunt-hq only read Greenhouse, Lever and -Ashby, so those four are written from one real board of each host, fetched -while this file was written (2026-09-23; Workable on 2026-09-28, when the -first board with roles was seen), rather than from the Go source. Teamtailor -is the same, written from live feeds on 2026-09-29. - -The body cap is not a straight port — see `MAX_BYTES` below. +`classify`'s nine functions and their keywords. The body cap is not a +straight port — see `MAX_BYTES` below. A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, -`lever:acme`, `teamtailor:acme`. Every board is fetched independently; one that fails or times +`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what +its slug is. Every board is fetched independently; one that fails or times out is logged, recorded as failing under `ats:failing:` (see `ATSSource.fetch`), and skipped, and the rest continue. -A Teamtailor slug is the `acme` of acme.teamtailor.com, or, for a career site -on the company's own domain, that whole host name: `teamtailor:careers.acme.com` -(see `_teamtailor_site`). - -Workable and SmartRecruiters page their list and state a total; Teamtailor -pages by offset and states none; every other host answers with the whole board -at once. A board with a total counts as complete only when it read as many -distinct roles as the total says, one without when it read up to an empty page -(see `_paged`). +A host that pages states a total or does not. A board with a total counts as +complete only when it read as many distinct roles as the total says, one +without when it read up to an empty page (see `_paged`); a host that answers +with the whole board at once is complete whenever it parses. """ from __future__ import annotations -import html +import ipaddress import json import logging import re +import sys import time -import xml.etree.ElementTree as ET -from collections.abc import Callable, Iterator -from datetime import UTC, datetime -from email.utils import parsedate_to_datetime +from collections.abc import Callable, Iterator, MutableMapping +from datetime import UTC, datetime, timedelta from typing import Any +from pathlib import Path +from urllib.parse import unquote import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, - SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html, + SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, ) +try: + from plugins import atshosts +except ModuleNotFoundError: + # `desk.sources.registry` loads this file by its path, under a name of its + # own, and the app puts the repo root on `sys.path` before it does; a + # registry run without the app (a script, another working directory) + # does not, and the host modules are found as the package beside this + # file. So the root goes on the path then, and only then. + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from plugins import atshosts +from plugins.atshosts._shared import Get, Row + +# The names the parsers had here before they moved to their host modules, +# kept for the callers and the tests that still reach for them on `ats`. +from plugins.atshosts._shared import ( # noqa: F401 + Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount, +) +from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401 +from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401 + log = logging.getLogger(__name__) #: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every -#: URL fetched here is `?content=true` against a fixed allow-list of seven ATS -#: API hosts. The real run of 2026-09-23 hit it on the biggest employers — +#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's +#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers — #: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and #: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised #: to 96 MB so those boards fit; a board that still exceeds it gets one retry -#: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least -#: the titles land. +#: at its host's `fallback_url` where it has one (Greenhouse, without +#: `?content=true`, see `_fetch_rows`), so at least the titles land. MAX_BYTES = 96 << 20 #: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai` @@ -84,96 +98,20 @@ MAX_BYTES = 96 << 20 TIMEOUTS = httpx.Timeout(10.0, read=30.0) DEADLINE = 60.0 -#: host -> (feed URL template, careers URL template). `{slug}` is filled in; -#: recruitee and personio put the slug in the hostname instead of the path, -#: and teamtailor's `{site}` is the whole host (see `_teamtailor_site`). -_BOARDS: dict[str, tuple[str, str]] = { - "greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs" - "?content=true&pay_transparency=true", - "https://job-boards.greenhouse.io/{slug}"), - "lever": ("https://api.lever.co/v0/postings/{slug}?mode=json", - "https://jobs.lever.co/{slug}"), - "ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}" - "?includeCompensation=true", - "https://jobs.ashbyhq.com/{slug}"), - "workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs", - "https://apply.workable.com/{slug}"), - "smartrecruiters": ("https://api.smartrecruiters.com/v1/companies/{slug}/postings", - "https://jobs.smartrecruiters.com/{slug}"), - "recruitee": ("https://{slug}.recruitee.com/api/offers/", - "https://{slug}.recruitee.com"), - "personio": ("https://{slug}.jobs.personio.de/xml", - "https://{slug}.jobs.personio.de"), - "teamtailor": ("https://{site}/jobs.rss", "https://{site}/jobs"), -} - #: host -> the name the host gives itself: the hosts this plugin reads, in #: the order a select offers them. What the Sources page's host select is #: rendered from, so it cannot offer a host `is_known_target` refuses. -HOSTS: dict[str, str] = { - "greenhouse": "Greenhouse", "lever": "Lever", "ashby": "Ashby", - "workable": "Workable", "smartrecruiters": "SmartRecruiters", - "recruitee": "Recruitee", "personio": "Personio", "teamtailor": "Teamtailor", -} -assert set(HOSTS) == set(_BOARDS) +HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()} + -#: The hosts a request or a redirect may land on for a given board host — a -#: fixed API host for five of them, the company's own subdomain for the two -#: that put the slug in the hostname, and the one site a Teamtailor slug names. -#: Anything else is refused, at the -#: first hop and at every redirect: a feed is external content, and a board -#: row must never be able to point this fetch at an internal address. def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: - fixed = { - "greenhouse": "boards-api.greenhouse.io", - "lever": "api.lever.co", - "ashby": "api.ashbyhq.com", - "workable": "apply.workable.com", - "smartrecruiters": "api.smartrecruiters.com", - } - if host in fixed: - want = fixed[host] - return lambda h: h == want - if host == "recruitee": - want = f"{slug}.recruitee.com" - return lambda h: h == want - if host == "personio": - want = f"{slug}.jobs.personio.de" - return lambda h: h == want - if host == "teamtailor": - want = _teamtailor_site(slug) - return lambda h: h == want - return lambda h: False - - -#: The host a Teamtailor career site is served from: `acme` is -#: acme.teamtailor.com, and a slug with a dot is a site on the company's own -#: domain, `careers.acme.com`, taken whole (the feed path is the same on both). -#: Only a DNS name with a letter in its last label is one, so an address -#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could be -#: pointed at; `_get` still refuses every host but this one, redirects included. -_TEAMTAILOR_HOST = re.compile(r"^[a-z0-9-]+(\.[a-z0-9-]+)*$") - - -def _teamtailor_site(slug: str) -> str: - slug = slug.strip().lower() - return slug if "." in slug else f"{slug}.teamtailor.com" - - -def _teamtailor_slug_ok(slug: str) -> bool: - site = _teamtailor_site(slug) - return bool(_TEAMTAILOR_HOST.match(site)) and not site.rsplit(".", 1)[-1].isdigit() - - -def _is_web_url(url: str) -> bool: - """A trimmed http(s) URL, or refuse — a feed is external content and a - `javascript:` or `data:` link must never reach an href. Port of - openings.go's `webURL`.""" - url = (url or "").strip() - if not url: - return False - parsed = httpx.URL(url) - return parsed.scheme in ("http", "https") and bool(parsed.host) + """The hosts a request or a redirect may land on for one board: what its + host module's `allowed` says, which is one fixed API host or the + company's own site. Anything else is refused, at the first hop and at + every redirect: a feed is external content, and a board row must never + be able to point this fetch at an internal address.""" + module = atshosts.HOSTS.get(host) + return module.allowed(slug) if module is not None else lambda h: False class _Refused(Exception): @@ -183,15 +121,54 @@ class _Refused(Exception): class _TooLarge(ValueError): """A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only knows the old shape still catches it; `_fetch_rows` catches it by type to - retry Greenhouse without `?content=true`.""" + retry at the host's `fallback_url`.""" + + +class _Disallowed(Exception): + """The site's robots.txt disallows the page a board is read from (see + `_check_robots`).""" + + +class _RobotsUnreachable(Exception): + """The site's robots.txt could not be read: a 5xx, a 429, a timeout, a + connection error or a redirect off the site. RFC 9309 says to assume + the site disallows everything then, so the board is not read this pull.""" + + +#: Host name suffixes that name a machine on a private network, never a +#: company's careers site. +_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa") + + +def _local_host(name: str) -> bool: + """Whether a host name is an address or a local name rather than a public + DNS name: an IP literal (v4 or v6), anything whose last label is a number + (`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the + resolver), `localhost`, or a private suffix. A Teamtailor slug can be any + host name, so the allow-list alone would let one point the fetch here.""" + name = name.strip("[]").rstrip(".").lower() + try: + ipaddress.ip_address(name) + return True + except ValueError: + pass + last = name.rsplit(".", 1)[-1] + return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last)) + or name == "localhost" or name.endswith(_LOCAL_SUFFIXES)) def _get(client: httpx.Client, url: str, host: str, slug: str, *, - method: str = "GET", payload: dict[str, Any] | None = None) -> bytes: - """Fetch one board feed: https only, the host's own allow-list, at most - three redirect hops each checked the same way, and the body cut off at - `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST - sends `payload` as JSON, `{}` when there is none. + method: str = "GET", payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes: + """Fetch one URL for a board: https only on the default port, never an + address or a local name (`_local_host`), the host's own allow-list, at + most three redirect hops each checked the same way, and the body cut off + at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST + sends `payload` as JSON, `{}` when there is none. `headers` are sent on + top of the User-Agent and Accept every request carries, and replace + either where they name it. With `www_twin`, a redirect may also land on + the first URL's host with `www.` added or taken off, which only the + robots.txt read asks for (see `_check_robots`). The body is streamed and counted as it arrives, so an oversized one is refused at `MAX_BYTES` rather than after it has been read whole, and @@ -200,15 +177,20 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, good. The deadline is checked between chunks, so it can be overrun by at most one read timeout.""" allow = _allowed_host(host, slug) + if www_twin: + bare = httpx.URL(url).host.removeprefix("www.") + board_allow = allow + allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731 + sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {}) deadline = time.monotonic() + DEADLINE current = url for _hop in range(4): parsed = httpx.URL(current) - if parsed.scheme != "https" or not allow(parsed.host): + if (parsed.scheme != "https" or parsed.port not in (None, 443) + or _local_host(parsed.host) or not allow(parsed.host)): raise _Refused(f"{current} is not an allowed host for {host}") with client.stream( - method, current, timeout=TIMEOUTS, follow_redirects=False, - headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"}, + method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent, json=(payload or {}) if method == "POST" else None, ) as response: if response.is_redirect: @@ -232,567 +214,27 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, raise ValueError("too many redirects") -def _parse_dt(value: Any) -> datetime | None: - """An aware datetime from whatever a board sends: ISO-8601 with an - offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC", - or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now".""" - if value is None: - return None - if isinstance(value, int | float): - if value <= 0: - return None - seconds = value / 1000 if value > 10**12 else value - return datetime.fromtimestamp(seconds, tz=UTC) - text = str(value).strip() - if not text: - return None - try: - parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) - except ValueError: - match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text) - if not match: - try: - parsed = parsedate_to_datetime(text) - except (TypeError, ValueError): - return None - return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) - parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00") - return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) +def _getter(client: httpx.Client, host: str, slug: str) -> Get: + """The `get` a host module is handed: `_get` bound to one board. `_get` + is looked up when a request is made, not when this is built, so a test + that replaces `ats._get` reaches every host; `payload` and `headers` are + passed on only when given, which keeps a replacement that predates them + working.""" + def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None) -> bytes: + extra: dict[str, Any] = {} + if payload is not None: + extra["payload"] = payload + if headers: + extra["headers"] = headers + return _get(client, url, host, slug, method=method, **extra) + return get -# ------------------------------------------------------------- the parsers - -#: What every parser turns a feed into: one dict per row, with these keys. -Row = dict[str, Any] - - -# ---------------------------------------------------------- structured pay -# -# A board's own salary range, mapped to yearly `salary_min`/`salary_max` -# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks -# `raw` into, per docs/decisions.md's "raw is transport, columns are -# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give -# structured pay on their list endpoint; Workable, SmartRecruiters and Teamtailor do not -# (checked live, 2026-09-29), so their parsers are untouched and `row.get` -# below reads a plain '' or None for them. - - -def _num(value: Any) -> float | None: - """A positive number from whatever a board sends (int, float, or a - numeral string), or None for anything else, including zero and a - negative — a board's placeholder 0 (an unfilled field, not a real - figure) must not become a salary floor of zero.""" - try: - number = float(value) - except (TypeError, ValueError): - return None - return number if number > 0 else None - - -def _yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any, - factors: dict[str, int]) -> tuple[int | None, int | None, str]: - """`(salary_min, salary_max, currency)` as yearly integers, or `(None, - None, "")` when there is no currency, no bound, or `interval` is not one - of `factors`' own keys — which is how an hourly or a daily rate is left - out rather than annualised into a number nobody stated. `factors` maps - the board's own interval spelling to the multiplier that makes it - yearly: 1 for a figure already yearly, 12 for a monthly one.""" - code = str(currency or "").strip().upper() - factor = factors.get(str(interval or "").strip()) - if not code or not factor: - return None, None, "" - lo, hi = _num(min_value), _num(max_value) - if lo is None and hi is None: - return None, None, "" - return (int(lo * factor) if lo is not None else None, - int(hi * factor) if hi is not None else None, code) - - -def _greenhouse_salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: - """The yearly range from `pay_input_ranges` (needs `pay_transparency=true` - on the feed URL, see `_BOARDS`), which the list endpoint already carries - in the same request — no per-role fetch. Only when there is exactly one - range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations", - one per office) gives no way to tell which office the role in hand is - at, and picking one would misrepresent the columns. Greenhouse's own - ranges are cents, and yearly by convention except an hourly or a daily - one, named as such in `title` with no separate field to check — those - are skipped rather than annualised into a number nobody stated.""" - if len(ranges) != 1: - return None, None, "" - entry = ranges[0] - title = str(entry.get("title") or "").lower() - if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")): - return None, None, "" - code = str(entry.get("currency_type") or "").strip().upper() - if not code: - return None, None, "" - lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents")) - if lo is None and hi is None: - return None, None, "" - return (int(lo / 100) if lo is not None else None, - int(hi / 100) if hi is not None else None, code) - - -def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: - """The yearly range from `compensation.summaryComponents` (needs - `?includeCompensation=true` on the feed URL, see `_BOARDS`): the first - `Salary` component, yearly or monthly (a monthly one multiplied by 12); - an hourly one is left out rather than annualised.""" - for component in components: - if component.get("compensationType") != "Salary": - continue - return _yearly_amount(component.get("minValue"), component.get("maxValue"), - component.get("currencyCode"), component.get("interval"), - {"1 YEAR": 1, "1 MONTH": 12}) - return None, None, "" - - -# ------------------------------------------------------------- salary_text -# -# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose: -# hourly and daily rates, a Greenhouse role with several pay zones, and a -# range with no currency are all dropped from those columns rather than -# guessed into a wrong yearly figure. `salary_text` is where that dropped -# information survives — one human-readable line per parser, written -# whether or not the pay conformed, so a person can read it and do their own -# sums and a stricter analysis can find the roles the columns left out -# (`Role.salary_state`, `app/models.py`). - -#: Formats a number the way a salary line writes it: no decimals for a whole -#: figure (the normal case for a yearly or monthly one), two for anything -#: else (an hourly or daily rate quoted with cents). -def _fmt_amount(value: float) -> str: - return f"{int(value):,}" if value == int(value) else f"{value:,.2f}" - - -def _bounds_words(lo: float | None, hi: float | None) -> str: - """"70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one - bound stated, or "" with neither.""" - if lo is not None and hi is not None: - return _fmt_amount(lo) if lo == hi else f"{_fmt_amount(lo)} to {_fmt_amount(hi)}" - if lo is not None: - return f"from {_fmt_amount(lo)}" - if hi is not None: - return f"up to {_fmt_amount(hi)}" - return "" - - -def _pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str: - """One statement of pay as a human-readable line — "EUR 70,000 to 90,000 - per year" — or "" when there is no bound to report at all. A currency - that was not stated does not drop the line, unlike the normalised - columns: it is said plainly instead, "80,000 to 100,000 per year (no - currency stated)", so the figure is not lost along with the currency.""" - bounds = _bounds_words(lo, hi) - if not bounds: - return "" - tail = f" per {period}" if period else "" - code = str(currency or "").strip().upper() - return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)" - - -def _greenhouse_salary_text(ranges: list[dict[str, Any]]) -> str: - """One line per `pay_input_ranges` entry, joined with "; " — every range - Greenhouse states, not only the single one `_greenhouse_salary` accepts - for the columns. With more than one, each is labelled with the board's - own `title` ("Zone 1 Pay Range: ..."), since that is the only way to - tell the ranges apart; with just the one, the label adds nothing and is - left off. The hour/day check is the same one `_greenhouse_salary` uses - to keep those out of the yearly columns; here it only picks the word - that follows "per".""" - multi = len(ranges) > 1 - lines = [] - for entry in ranges: - title = str(entry.get("title") or "").strip() - low_title = title.lower() - period = ("hour" if any(w in low_title for w in ("hour", "/hr")) - else "day" if "day" in low_title else "year") - lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents")) - line = _pay_line(lo / 100 if lo is not None else None, - hi / 100 if hi is not None else None, - entry.get("currency_type"), period) - if not line: - continue - if multi: - label = title.rstrip(":").strip() or "Range" - line = f"{label}: {line}" - lines.append(line) - return "; ".join(lines) - - -def _lever_salary_text(salary_range: dict[str, Any]) -> str: - """One line from `salaryRange`, whatever its `interval` — including an - hourly or daily one, which `_yearly_amount` drops from the columns. - Lever's interval is "per--" ("per-year-salary", - "per-hour-wage"); the middle word is the period.""" - interval = str(salary_range.get("interval") or "") - parts = interval.split("-") - period = parts[1] if len(parts) >= 3 else "" - return _pay_line(_num(salary_range.get("min")), _num(salary_range.get("max")), - salary_range.get("currency"), period) - - -def _ashby_salary_text(compensation: dict[str, Any]) -> str: - """One line per `summaryComponents` entry that states a bound, labelled - by the board's own `compensationType` when more than one does (a salary - range beside a bonus range, say); with just the one, the label is left - off, the same as `_greenhouse_salary_text`. `interval` is " - " ("1 YEAR", "1 HOUR"); the last word, lowered, is the period. - With nothing to build a line from, Ashby's own prose summary - (`compensationTierSummary`, else `scrapeableCompensationSalarySummary`) - is what is left to show.""" - components = compensation.get("summaryComponents") or [] - stated = [c for c in components if _num(c.get("minValue")) is not None - or _num(c.get("maxValue")) is not None] - lines = [] - for component in stated: - period = str(component.get("interval") or "").split(" ")[-1].lower() - line = _pay_line(_num(component.get("minValue")), _num(component.get("maxValue")), - component.get("currencyCode"), period) - if not line: - continue - if len(stated) > 1: - label = str(component.get("compensationType") or "").strip() - line = f"{label}: {line}" if label else line - lines.append(line) - if lines: - return "; ".join(lines) - return str(compensation.get("compensationTierSummary") - or compensation.get("scrapeableCompensationSalarySummary") or "") - - -def _recruitee_salary_text(salary: dict[str, Any]) -> str: - """One line from Recruitee's `salary` object; `period` is already a - plain word ("year", "month").""" - period = str(salary.get("period") or "").strip().lower() - return _pay_line(_num(salary.get("min")), _num(salary.get("max")), - salary.get("currency"), period) - - -#: Personio's `salaryInformation/type` spellings, to the plain period word -#: `_pay_line` wants ("yearly" -> "year"). Anything else is passed through. -_PERSONIO_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour", - "daily": "day", "weekly": "week"} - - -def _personio_salary_text(salary_info: ET.Element) -> str: - """One line from a `` element, whatever its `type`, - including an hourly or daily one that `_yearly_amount` drops from the - columns.""" - period_raw = (salary_info.findtext("type") or "").strip().lower() - period = _PERSONIO_PERIODS.get(period_raw, period_raw) - return _pay_line(_num(salary_info.findtext("min")), _num(salary_info.findtext("max")), - salary_info.findtext("currencyCode"), period) - - -def _parse_greenhouse(body: bytes) -> list[Row]: - data = json.loads(body) - rows = [] - for j in data.get("jobs") or []: - title = (j.get("title") or "").strip() - url = (j.get("absolute_url") or "").strip() - if not title or not _is_web_url(url): - continue - departments = j.get("departments") or [] - # Greenhouse ships the description HTML-entity-escaped inside the - # JSON string (`<p>`), so it needs unescaping before the tags - # themselves can be stripped. - content = html.unescape(j.get("content") or "") - ranges = j.get("pay_input_ranges") or [] - salary_min, salary_max, currency = _greenhouse_salary(ranges) - rows.append({ - "source_id": str(j.get("id") or ""), - "title": title, - "location": ((j.get("location") or {}).get("name") or "").strip(), - "url": url, - "posted": _parse_dt(j.get("first_published") or j.get("updated_at")), - "department": (departments[0].get("name") if departments else "") or "", - "description": strip_html(content), - "remote": None, # Greenhouse's feed does not say - "salary_text": _greenhouse_salary_text(ranges), - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -def _parse_lever(body: bytes) -> list[Row]: - data = json.loads(body) - if not isinstance(data, list): - return [] - rows = [] - for j in data: - title = (j.get("text") or "").strip() - url = (j.get("hostedUrl") or "").strip() - if not title or not _is_web_url(url): - continue - categories = j.get("categories") or {} - description = j.get("description") or j.get("descriptionPlain") or "" - salary_range = j.get("salaryRange") or {} - salary_min, salary_max, currency = _yearly_amount( - salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"), - salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12}) - rows.append({ - "source_id": str(j.get("id") or ""), - "title": title, - "location": (categories.get("location") or "").strip(), - "url": url, - "posted": _parse_dt(j.get("createdAt")), - "department": (categories.get("department") or categories.get("team") or "").strip(), - "description": strip_html(description), - "remote": (j.get("workplaceType") or "").strip() or None, - "salary_text": _lever_salary_text(salary_range), - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -def _parse_ashby(body: bytes) -> list[Row]: - data = json.loads(body) - rows = [] - for j in data.get("jobs") or []: - title = (j.get("title") or "").strip() - url = (j.get("jobUrl") or j.get("applyUrl") or "").strip() - if not title or not _is_web_url(url): - continue - description = j.get("descriptionHtml") or j.get("descriptionPlain") or "" - compensation = j.get("compensation") or {} - salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or []) - rows.append({ - "source_id": str(j.get("id") or ""), - "title": title, - "location": (j.get("location") or "").strip(), - "url": url, - "posted": _parse_dt(j.get("publishedAt")), - "department": (j.get("department") or j.get("team") or "").strip(), - "description": strip_html(description), - "remote": j.get("isRemote"), - "salary_text": _ashby_salary_text(compensation), - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -def _parse_workable(body: bytes, slug: str) -> list[Row]: - return _workable_rows(json.loads(body), slug) - - -def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]: - """One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs - -> `{"total": int, "results": [...], "nextPage": token}`, as Hugging - Face's board answered on 2026-09-28. Ten results a page; the next page is - the same POST with `{"token": nextPage}`, and the last page has no - `nextPage` (see `_paged`). A result carries no URL and no text: the job page is - `https://apply.workable.com/{slug}/j/{shortcode}/`, and the description - is only on the per-job endpoint, one request per role, so `description` - stays empty here as it does for SmartRecruiters. `location` is an object - and `department` a list; `remote` is a flag beside `workplace` - (remote, hybrid, on_site).""" - rows = [] - for j in data.get("results") or []: - title = (j.get("title") or "").strip() - shortcode = (j.get("shortcode") or "").strip() - if not title or not shortcode: - continue - loc = j.get("location") or {} - location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p) - dept = j.get("department") or "" - if isinstance(dept, list): - dept = ", ".join(d for d in dept if d) - rows.append({ - "source_id": str(j.get("id") or shortcode), - "title": title, - "location": location, - "url": f"https://apply.workable.com/{slug}/j/{shortcode}/", - "posted": _parse_dt(j.get("published")), - "department": dept.strip(), - "description": "", - "remote": bool(j.get("remote")) or j.get("workplace") == "remote", - "salary_text": "", # Workable's list endpoint states no pay at all - }) - return rows - - -def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]: - return _smartrecruiters_rows(json.loads(body), slug) - - -def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]: - """One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings - -> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a - page, whatever `limit` asks for; the next is `?offset=` further on (see - `_paged`). The list omits the job description (only the - per-posting detail endpoint has it, and fetching one per row would be - hundreds of extra requests per board), so `description` stays empty here.""" - rows = [] - for j in data.get("content") or []: - title = (j.get("name") or "").strip() - job_id = str(j.get("id") or "") - if not title or not job_id: - continue - location = j.get("location") or {} - loc_text = (location.get("fullLocation") - or ", ".join(p for p in (location.get("city"), location.get("country")) if p)) - rows.append({ - "source_id": job_id, - "title": title, - "location": (loc_text or "").strip(), - "url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}", - "posted": _parse_dt(j.get("releasedDate")), - "department": ((j.get("department") or {}).get("label") or "").strip(), - "description": "", - "remote": location.get("remote"), - "salary_text": "", # SmartRecruiters' list endpoint states no pay at all - }) - return rows - - -def _parse_recruitee(body: bytes) -> list[Row]: - data = json.loads(body) - rows = [] - for j in data.get("offers") or []: - title = (j.get("title") or "").strip() - url = (j.get("careers_url") or "").strip() - if not title or not _is_web_url(url): - continue - salary = j.get("salary") if isinstance(j.get("salary"), dict) else {} - salary_min, salary_max, currency = _yearly_amount( - salary.get("min"), salary.get("max"), salary.get("currency"), - salary.get("period"), {"year": 1, "month": 12}) - rows.append({ - "source_id": str(j.get("id") or ""), - "title": title, - "location": (j.get("location") or j.get("city") or "").strip(), - "url": url, - "posted": _parse_dt(j.get("published_at") or j.get("created_at")), - "department": (j.get("department") or "").strip(), - "description": strip_html(j.get("description") or ""), - "remote": j.get("remote"), - "salary_text": _recruitee_salary_text(salary), - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -def _parse_personio(body: bytes, slug: str) -> list[Row]: - """`{slug}.jobs.personio.de/xml` -> `...`. No job - URL in the feed itself; Personio's own page is `/job/{id}`, confirmed - live against a real posting (2026-09-23).""" - root = ET.fromstring(body) - rows = [] - for position in root.findall("position"): - job_id = (position.findtext("id") or "").strip() - title = (position.findtext("name") or "").strip() - if not job_id or not title: - continue - description = " ".join( - (value.text or "") for value in position.findall("jobDescriptions/jobDescription/value")) - salary_info = position.find("salaryInformation") - salary_min = salary_max = None - currency = "" - salary_text = "" - if salary_info is not None: - salary_min, salary_max, currency = _yearly_amount( - salary_info.findtext("min"), salary_info.findtext("max"), - salary_info.findtext("currencyCode"), salary_info.findtext("type"), - {"yearly": 1, "monthly": 12}) - salary_text = _personio_salary_text(salary_info) - rows.append({ - "source_id": job_id, - "title": title, - "location": (position.findtext("office") or "").strip(), - "url": f"https://{slug}.jobs.personio.de/job/{job_id}", - "posted": _parse_dt(position.findtext("createdAt")), - "department": (position.findtext("department") or "").strip(), - "description": strip_html(description), - "remote": "remote" in (position.findtext("office") or "").lower() or None, - "salary_text": salary_text, - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`. -_TT = "{https://teamtailor.com/locations}" - -#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing -#: about where the role is worked and `temporary` is remote for a spell only, -#: so both stay unknown rather than become "onsite" or "remote". -_TEAMTAILOR_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"} - - -def _parse_teamtailor(body: bytes, slug: str) -> list[Row]: - return _teamtailor_rows(_teamtailor_items(body), slug) - - -def _teamtailor_items(body: bytes) -> list[ET.Element]: - """`{site}/jobs.rss` -> `...`, the feed every Teamtailor - career site serves (checked live on a dozen sites, 2026-09-29). A missing - `channel` is a page that is not a feed, and raises.""" - channel = ET.fromstring(body).find("channel") - if channel is None: - raise ValueError("not a Teamtailor feed: no channel") - return channel.findall("item") - - -def _teamtailor_rows(items: list[ET.Element], slug: str) -> list[Row]: - """One page of items. The item has the title, the job URL (on the site's - own domain), a `guid` uuid, `pubDate`, the HTML `description` in full, - `remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the - `tt:` namespace, `locations` (a list: `city` and `country` each, `name` free - text), `department` and `role`, either of which may be empty. `place` joins - the distinct locations with "; ", which `country_of` reads a location at a - time. No pay is stated anywhere in the feed.""" - rows = [] - for item in items: - title = (item.findtext("title") or "").strip() - url = (item.findtext("link") or "").strip() - if not title or not _is_web_url(url): - continue - places: list[str] = [] - for loc in item.findall(f"{_TT}locations/{_TT}location"): - city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip() - country = (loc.findtext(f"{_TT}country") or "").strip() - place = city if country == city else ", ".join(p for p in (city, country) if p) - if place and place not in places: - places.append(place) - rows.append({ - "source_id": (item.findtext("guid") or url).strip(), - "title": title, - "location": "; ".join(places), - "url": url, - "posted": _parse_dt(item.findtext("pubDate")), - "department": ((item.findtext(f"{_TT}department") or "").strip() - or (item.findtext(f"{_TT}role") or "").strip()), - "description": html.unescape(strip_html(item.findtext("description") or "")), - "remote": _TEAMTAILOR_REMOTE.get((item.findtext("remoteStatus") or "").strip()), - "salary_text": "", # Teamtailor's feed states no pay at all - }) - return rows - - -#: host -> parser. Every one takes the raw response body, plus the slug for -#: the hosts whose feed does not carry the URL or the company itself. -_PARSERS: dict[str, Callable[..., list[Row]]] = { - "greenhouse": lambda body, slug: _parse_greenhouse(body), - "lever": lambda body, slug: _parse_lever(body), - "ashby": lambda body, slug: _parse_ashby(body), - "workable": _parse_workable, - "smartrecruiters": _parse_smartrecruiters, - "recruitee": lambda body, slug: _parse_recruitee(body), - "personio": _parse_personio, - "teamtailor": _parse_teamtailor, -} +#: host -> parser, for the hosts whose module parses a whole body. Every one +#: takes the raw response body and the slug. +_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { + name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")} # --------------------------------------------------------- the classifier @@ -923,98 +365,47 @@ def classify(department: str, title: str) -> str: def is_known_target(target: str) -> bool: - """`host:slug`, with `host` one of `_BOARDS` and a non-empty slug (for - Teamtailor one that names a host, see `_teamtailor_slug_ok`). What - the target validator refuses: `app companies add` and the pull job - both call this before touching the network.""" + """`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's + `slug_ok` accepts. What the target validator refuses: `app companies add` + and the pull job both call this before touching the network.""" host, sep, slug = target.partition(":") - return (bool(sep) and host in _BOARDS and bool(slug.strip()) - and (host != "teamtailor" or _teamtailor_slug_ok(slug))) + module = atshosts.HOSTS.get(host) + return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug) def careers_url(target: str) -> str: """The company's own careers page for `host:slug`, or "" for an unknown host. What `app companies add` stores alongside the board.""" host, _, slug = target.partition(":") - if host not in _BOARDS: - return "" - return _BOARDS[host][1].format(slug=slug, site=_teamtailor_site(slug)) + module = atshosts.HOSTS.get(host) + return module.careers_url(slug) if module is not None else "" #: Pages read from one paged board at most, and the pause between two of #: them. Workable answers ten roles a page, SmartRecruiters a hundred and -#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause keeps Workable, which answers -#: a burst with 429, from seeing one. +#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause +#: keeps Workable, which answers a burst with 429, from seeing one. The same +#: pause separates two detail reads. MAX_PAGES = 100 PAGE_DELAY = 0.5 -#: SmartRecruiters' page size, which is also the most it serves per page. -SMARTRECRUITERS_PAGE = 100 - -#: Teamtailor's page size, asked for as `per_page`. A feed with neither -#: parameter answers with every role (73 of 73 at Shine, 2026-09-29), but says -#: nowhere that it is the whole list, so this reads by page to an empty one; -#: `per_page=200` was answered in full too, and the cursor advances by what a -#: page returned, so a smaller cap on the host's side would not skip a role. -TEAMTAILOR_PAGE = 50 - -#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its -#: raw results, the total it states (None for a host that states none), and the -#: cursor of the next page (None after the last). The first call gets cursor -#: None. -Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any] - - -def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page: - url = _BOARDS["workable"][0].format(slug=slug) - data = json.loads(_get(client, url, "workable", slug, method="POST", - payload={"token": cursor} if cursor else {})) - return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None - - -def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page: - offset = int(cursor or 0) - url = (_BOARDS["smartrecruiters"][0].format(slug=slug) - + f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}") - data = json.loads(_get(client, url, "smartrecruiters", slug)) - content = data.get("content") or [] - return data, content, data.get("totalFound"), offset + len(content) if content else None - - -def _teamtailor_page(client: httpx.Client, slug: str, cursor: Any) -> Page: - offset = int(cursor or 0) - url = (_BOARDS["teamtailor"][0].format(site=_teamtailor_site(slug)) - + f"?per_page={TEAMTAILOR_PAGE}&offset={offset}") - items = _teamtailor_items(_get(client, url, "teamtailor", slug)) - results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items] - return items, results, None, offset + len(items) if items else None - - -#: host -> (one page, that page's rows) for the hosts that page. -_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page], - Callable[[dict[str, Any], str], list[Row]]]] = { - "workable": (_workable_page, _workable_rows), - "smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows), - "teamtailor": (_teamtailor_page, _teamtailor_rows), -} - def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: - """Every page of a Workable, SmartRecruiters or Teamtailor board, - `PAGE_DELAY` apart, until the host has no next page, the stated total is - reached, or `MAX_PAGES`. Returns the rows and whether the board is - complete: as many distinct results read as the host's stated total, or, - for a host that states none (Teamtailor), the read ended on an empty page - (a page with nothing new in it ends the read short of that, incomplete, - so an offset the host ignores cannot loop). Counted on the raw - results, so a row the parser drops does not make a board incomplete, and - on distinct ids, so a page that shifted under an offset and repeated a - role does not make it complete. + """Every page of a board whose host pages, `PAGE_DELAY` apart, until the + host has no next page, the stated total is reached, or `MAX_PAGES`. + Returns the rows and whether the board is complete: as many distinct + results read as the host's stated total, or, for a host that states none + (Teamtailor), the read ended on an empty page (a page with nothing new in + it ends the read short of that, incomplete, so an offset the host ignores + cannot loop). Counted on the raw results, so a row the parser drops does + not make a board incomplete, and on distinct ids, so a page that shifted + under an offset and repeated a role does not make it complete. A 429 after the first page stops the read with what it has, incomplete; one on the first page raises like any other failure.""" host, _, slug = target.partition(":") - page_of, rows_of = _PAGED[host] + module = atshosts.HOSTS[host] + get = _getter(client, host, slug) rows: dict[str, Row] = {} seen: set[str] = set() total: Any = None @@ -1023,7 +414,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: if page: time.sleep(PAGE_DELAY) try: - data, results, total, cursor = page_of(client, slug, cursor) + data, results, total, cursor = module.page(get, slug, cursor) except httpx.HTTPStatusError as exc: if not page or exc.response.status_code != 429: raise @@ -1032,7 +423,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: return list(rows.values()), False before = len(seen) seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results} - for row in rows_of(data, slug): + for row in module.rows(data, slug): rows.setdefault(row["source_id"], row) if (cursor is None or (isinstance(total, int) and len(seen) >= total) or (total is None and len(seen) == before)): @@ -1045,45 +436,284 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: - """One board, fetched and parsed: its rows and whether they are the whole - board. Raises on any failure — refused host, timeout, a feed shape none - of the parsers understand. A host that does not page answers with the - whole board, so it is complete whenever it parses; a paged one is - complete when `_paged` says so. + """One board's listing, fetched and parsed: its rows and whether they are + the whole board. Raises on any failure — refused host, timeout, a feed + shape the host's parser does not understand. A host that does not page + answers with the whole board, so it is complete whenever it parses; a + paged one is complete when `_paged` says so. - A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES` - gets one retry without it: the description is lost, but the titles land - rather than the whole board being skipped.""" + A body that exceeds `MAX_BYTES` gets one retry at the host's + `fallback_url` where it has one (Greenhouse, without `?content=true`): + the description is lost, but the titles land rather than the whole board + being skipped.""" if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") host, _, slug = target.partition(":") - if host in _PAGED: + module = atshosts.HOSTS[host] + if hasattr(module, "page"): return _paged(client, target) - url = _BOARDS[host][0].format(slug=slug) + get = _getter(client, host, slug) try: - body = _get(client, url, host, slug, method="GET") + body = get(module.feed_url(slug)) except _TooLarge: - if host != "greenhouse" or "?" not in url: + if not hasattr(module, "fallback_url"): raise - log.warning("ats: %s: body exceeded %d bytes with content=true; " - "retrying without it", target, MAX_BYTES) - url = url.split("?", 1)[0] - body = _get(client, url, host, slug, method="GET") - return _PARSERS[host](body, slug), True + log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed", + target, MAX_BYTES) + body = get(module.fallback_url(slug)) + return module.parse(body, slug), True + + +# -------------------------------------------------------------- robots.txt + +#: How long a robots.txt answer holds for one board before it is read again. +ROBOTS_TTL = timedelta(days=1) + + +#: This app's product token, the name a robots.txt group addresses it by: +#: the `desk` of `desk/ (personal aggregator)`. +ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower() + +#: One rule of a group: `(allow, path pattern)`. +_Rule = tuple[bool, str] + + +def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]: + """The groups of a robots.txt, per RFC 9309: one or more `user-agent` + lines (their tokens lowercased) and the `allow` and `disallow` rules that + follow them, up to the next `user-agent` line after a rule. Comments, + other keys and rules before any `user-agent` line are left out, and so is + an empty rule, which allows everything and so says nothing.""" + groups: list[tuple[set[str], list[_Rule]]] = [] + in_agents = False + for line in text.splitlines(): + key, sep, value = line.split("#", 1)[0].partition(":") + key, value = key.strip().lower(), value.strip() + if not sep: + continue + if key == "user-agent": + if not in_agents: + groups.append((set(), [])) + in_agents = True + groups[-1][0].add(value.lower()) + elif key in ("allow", "disallow") and groups: + in_agents = False + if value: + groups[-1][1].append((key == "allow", value)) + return groups + + +def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]: + """The rules for `token`: every group that names it (matched whole and + ignoring case, as RFC 9309 says, not as a substring), merged; else every + `*` group, merged; else none, which allows everything.""" + for wanted in (token, "*"): + rules = [rule for agents, group in groups if wanted in agents for rule in group] + if rules or any(wanted in agents for agents, _group in groups): + return rules + return [] + + +def _robots_pattern(path: str) -> re.Pattern[str]: + """A rule's path as a regex anchored at the start: `*` is any run of + characters, and a `$` at the end anchors the end.""" + anchored = path.endswith("$") + body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*")) + return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL) + + +def _robots_verdict(rules: list[_Rule], path: str) -> bool: + """Whether `path` is allowed by `rules`: the longest matching rule wins, + an allow wins a tie, and no match allows. Both sides are compared + percent-decoded, so `%7E` and `~` are one character.""" + path = unquote(path) + best: tuple[int, bool] | None = None + for allow, pattern in rules: + if _robots_pattern(unquote(pattern)).match(path): + weight = (len(pattern), allow) + best = weight if best is None or weight > best else best + return best is None or best[1] + + +def _robots_allows(body: bytes, page: str) -> bool: + """Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules + for every agent (`*`), and by those it has for this app's own token + (`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this + app by name is honoured as well. Paths are case-sensitive; a byte order + mark is dropped.""" + groups = _robots_groups(body.decode("utf-8-sig", "replace")) + url = httpx.URL(page) + path = (url.raw_path or b"/").decode("ascii", "replace") + return all(_robots_verdict(_robots_rules(groups, token), path) + for token in ("*", ROBOTS_TOKEN)) + + +def _check_robots(client: httpx.Client, target: str, + state: MutableMapping[str, str] | None = None) -> None: + """Raise `_Disallowed` when the site of a host with a `robots_page` has a + robots.txt that disallows that page, and `_RobotsUnreachable` when it + cannot be read. Read through `_get`, so from the board's own allowed + hosts only, except that a redirect between the host and its `www.` twin + is followed (at most three hops, https only); any other redirect off the + site is unreachable. With `state`, the verdict and when it was reached + are kept under `ats:robots:` and held for `ROBOTS_TTL`. + + By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey, + which allows, and is kept like any other verdict. A 5xx, a 429, a + timeout or a connection error means the file is unreachable, which the + RFC says to treat as disallowing everything: the board is not read on + this pull, fails with "robots.txt could not be read", and nothing is + kept, so the next pull asks again. A 429 counts as unreachable although + it is a 4xx, since it says "not now" rather than "there is none".""" + host, _, slug = target.partition(":") + module = atshosts.HOSTS.get(host) + if module is None or not hasattr(module, "robots_page") or not is_known_target(target): + return + page = module.robots_page(slug) + key = f"ats:robots:{target}" + now = datetime.now(UTC) + if state is not None: + try: + kept = json.loads(state.get(key) or "{}") + fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL + except (ValueError, KeyError, TypeError): + kept, fresh = {}, False + if fresh: + if not kept.get("allowed", True): + raise _Disallowed(f"robots.txt disallows {page}") + return + url = f"https://{httpx.URL(page).host}/robots.txt" + try: + body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True) + except httpx.HTTPStatusError as exc: + code = exc.response.status_code + if not 400 <= code < 500 or code == 429: + raise _RobotsUnreachable(f"robots.txt answered {code}") from exc + allowed = True + except Exception as exc: + raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc + else: + allowed = _robots_allows(body, page) + if state is not None: + state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"), + "allowed": allowed}) + if not allowed: + raise _Disallowed(f"robots.txt disallows {page}") + + +# ---------------------------------------------------------------- details + +#: Roles read in detail from one board in one pull at most. The rest of a +#: large new board is read over the next pulls, 200 at a time. +DETAIL_CAP = 200 + +#: The row keys a host's `detail` may fill: those whose column survives a +#: later pull that sends the listing's empty value for them, so what a role's +#: own page said is not wiped the pull after it was read. `description`, +#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are +#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only +#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the +#: earliest date a role ever had. `title` and `url` are not here, since the +#: listing always states them and would overwrite them; nor are `department` +#: (no column of its own, only `raw` and the classifier's guess) and +#: `salary_text`, both rewritten from the listing on every pull. +DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min", + "salary_max", "posted") + + +def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool, + detailed: set[str], failed: set[str] + ) -> tuple[list[Row], set[str], set[str]]: + """The rows of a board whose host has `detail`, with the roles not yet + read in detail filled from their own pages: at most `DETAIL_CAP` of them, + `PAGE_DELAY` apart, first the roles never tried and then those whose + detail failed before (`failed`), so a few roles that always fail cannot + take the whole cap pull after pull. Returns the rows and the compound ids + read (for `ats:detailed:`) and failed (for + `ats:detailfailed:`). + + Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key + is left out with a debug line. Every row not read in detail in this call + goes out with an empty description: a role read on an earlier pull keeps + the one it has, since `upsert_role` never overwrites a filled column with + an empty value, and a teaser from the listing must not replace it. + + A detail that answers 404 or 410 counts as read: the role is gone or has + no page of its own, and asking again would not change that. One that + raises otherwise is logged and the role kept in `failed`; a 429 stops the + reads for this board until the next pull, and does not mark the role as + failing. None of this touches completeness, which is the listing's alone. + The ids of roles no longer listed are dropped from both sets, but only + after a complete listing, so one cut short does not make the next pull + read its missing roles again.""" + host, _, slug = target.partition(":") + module = atshosts.HOSTS[host] + get = _getter(client, host, slug) + done, failing = set(detailed), set(failed) - set(detailed) + + def compound(row: Row) -> str: + return f"{host}:{slug}:{row['source_id']}" + + unread = [row for row in rows if compound(row) not in done] + queue = ([row for row in unread if compound(row) not in failing] + + [row for row in unread if compound(row) in failing])[:DETAIL_CAP] + filled: dict[str, Row] = {} + for row in queue: + key = compound(row) + time.sleep(PAGE_DELAY) + try: + answer = module.detail(get, slug, row) + except httpx.HTTPStatusError as exc: + code = exc.response.status_code + if code == 429: + log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull", + target) + break + if code in (404, 410): + done.add(key) + failing.discard(key) + else: + log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code) + failing.add(key) + continue + except Exception as exc: + log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc) + failing.add(key) + continue + if ignored := sorted(set(answer) - set(DETAIL_FIELDS)): + log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS", + target, row["source_id"], ", ".join(ignored)) + filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} + done.add(key) + failing.discard(key) + out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows] + if complete: + listed = {compound(row) for row in rows} + done &= listed + failing &= listed + return out, done, failing def fetch_board(target: str) -> tuple[list[Row], bool]: """Fetch and parse one board directly: no `Source`, no `self.state`, just "does this board answer, what does it list, and is that all of it". For `app companies add|check`, which ask that question about one board at a - time outside any pull. Raises the same way `_fetch_rows` does.""" + time outside any pull. The listing only, with no detail reads, and + robots.txt read afresh. Raises the same way `_fetch_rows` does, and + `_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or + that cannot be checked.""" with httpx.Client() as client: + _check_robots(client, target) return _fetch_rows(client, target) def failure_reason(exc: BaseException) -> str: """One short line for why a board did not answer: the status of an HTTP error, else the exception's class and the first line of its message.""" + if isinstance(exc, _Disallowed): + return "robots.txt disallows it" + if isinstance(exc, _RobotsUnreachable): + return "robots.txt could not be read" if isinstance(exc, httpx.HTTPStatusError): return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() lines = str(exc).strip().splitlines() @@ -1095,8 +725,7 @@ def failure_reason(exc: BaseException) -> str: class ATSSource(Source): name = "ats" - description = "Company job boards: Greenhouse, Lever, Ashby, Workable, " \ - "SmartRecruiters, Recruitee, Personio, Teamtailor." + description = f"Company job boards: {', '.join(HOSTS.values())}." #: The raw entries under `sources.ats.boards` are `{"url": target, "label": #: company}` — `url` carries the `host:slug` target and `label` the #: company name, which is how `Source.targets()`'s generic dict reading @@ -1116,6 +745,14 @@ class ATSSource(Source): )}, ) + def _id_set(self, key: str) -> set[str]: + """A JSON list of ids kept in state, as a set; empty when there is + none or it does not parse.""" + try: + return {str(i) for i in json.loads(self.state.get(key) or "[]")} + except (ValueError, TypeError): + return set() + def fetch(self) -> Iterator[Item]: with httpx.Client() as client: # Handed in by the batch that runs this plugin, as the marker of @@ -1130,6 +767,7 @@ class ATSSource(Source): host, _, slug = target.partition(":") failing = f"ats:failing:{target}" try: + _check_robots(client, target, self.state) rows, complete = _fetch_rows(client, target) except Exception as exc: log.warning("ats: %s failed (%s)", target, exc) @@ -1146,6 +784,14 @@ class ATSSource(Source): continue self.state.pop(failing, None) + detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}") + detail_sets: tuple[set[str], set[str]] | None = None + if hasattr(atshosts.HOSTS.get(host), "detail"): + detailed, failed = (self._id_set(key) for key in detail_keys) + rows, detailed, failed = _read_details(client, target, rows, complete, + detailed, failed) + detail_sets = (detailed, failed) + seen_ids = [] for row in rows: # The compound id, matching `roles.source_id` — what @@ -1169,11 +815,10 @@ class ATSSource(Source): "function": classify(row["department"], row["title"]), "remote": row["remote"], # Every parser sets salary_text, empty or not (see - # "salary_text" above); only Ashby, Greenhouse, - # Lever, Recruitee and Personio's also set the - # normalised figures (see "structured pay" - # above), so the rest leave them at row.get's - # default of no figure. + # "salary_text" in `atshosts/_shared.py`); only + # the hosts that state structured pay also set the + # normalised figures, so the rest leave them at + # row.get's default of no figure. "salary_text": row.get("salary_text", ""), "salary_min": row.get("salary_min"), "salary_max": row.get("salary_max"), @@ -1186,7 +831,13 @@ class ATSSource(Source): # board that blew up half way through is neither marked ok nor # has a partial id list treated as the whole of it. `complete` # says whether the ids are the whole board, which the pull - # needs before it closes what they leave out. + # needs before it closes what they leave out. The ids read in + # detail are kept here too, after every row went out, so a + # pull stopped half way reads the rest again rather than + # losing their descriptions. self.state[f"ats:ok:{target}"] = run_marker self.state[f"ats:complete:{target}"] = "1" if complete else "" self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) + if detail_sets is not None: + for key, ids in zip(detail_keys, detail_sets, strict=True): + self.state[key] = json.dumps(sorted(ids)) diff --git a/plugins/atshosts/__init__.py b/plugins/atshosts/__init__.py new file mode 100644 index 0000000..efe0623 --- /dev/null +++ b/plugins/atshosts/__init__.py @@ -0,0 +1,210 @@ +"""The hosts the `ats` plugin reads, one module each. + +A host registers by existing: every module in this package whose name does +not start with `_` is loaded when the package is imported, checked against +the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A +module that breaks the contract fails the import with an ImportError naming +the module and what it lacks, rather than a pull that fails board by board. +`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host +select and its description from `HOSTS`, and `app/jobs/discover.py` its +patterns and probes. The package sits beside `ats.py` rather than being +`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and +the tests patch names on the `ats` module. + +A module is named after its host (`plugins/atshosts/workday.py`) and imports +`desk` and `._shared` only, never `plugins.ats`. It holds: + + NAME = "workday" # the host of a target `workday:`; equals the module's name + LABEL = "Workday" # what the Sources page's select shows + PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE + RANK = 100 # optional int; lower comes first in the select and the probes + KEEP_CASE = False # optional; True keeps a discovered slug's case + + def slug_ok(slug: str) -> bool # strict, see below + def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach + def careers_url(slug: str) -> str # the page a person opens + +and one of two ways to read a board: + + def feed_url(slug: str) -> str # the whole board in one answer, + def parse(body: bytes, slug: str) -> list[Row] # read with `parse` + + def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call, + def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page + +and, optionally: + + def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read + # once when `feed_url` is over the cap + def robots_page(slug: str) -> str # the URL robots.txt must allow + def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page + def probe_url(slug: str) -> str # discovery: 2xx only for a board + def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough + def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as + +**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts +hold 10 to 80, which keeps the order they had before there were modules, so +a new host comes after them unless it says otherwise. The order decides the +select's default and which host wins when discovery's guessed slug exists on +two. + +**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`, +which lets a trailing newline through) and refuses anything that could add a +path, a query, a fragment, a port or a user to the URLs built from it. +`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok` +(one DNS label) cover the usual cases; a slug with several parts joins them +with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`). + +**Patterns.** The slug is group 1, or, when a pattern has more than one +group, the groups that matched joined by "/". Only group 1 is checked +against discovery's words that name no board (`api`, `jobs`, `embed`, ...). +The slug is lowercased unless `KEEP_CASE` is True: set it when the host's +paths are case-sensitive (Workday's site), since robots.txt paths are too. A +pattern without a group names the host with no slug. A host that +`discover.py` names in its own `_UNREADABLE` table is taken from its module +instead as soon as the module exists. + +**Fetching.** `allowed(slug)` returns a predicate over a lowercased host +name; `_shared.only(host)` is the usual one. Every request goes through +`get`, which is `ats._get` bound to the board: + + get(url, *, method="GET", payload=None, headers=None) -> bytes + +A POST sends `payload` as JSON (`{}` when None); `headers` are added to the +default User-Agent and `Accept: application/json, text/xml`, and replace +either where they name it. `get` is https only on port 443, never to an IP +address or a local name (`localhost`, `.local`, `.internal`, ...), follows at +most three redirects each checked against `allowed(slug)`, cuts the body at +`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns +the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read +`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection +failure, and another exception (`ValueError` among them) for a refused host, +an oversized body or a broken redirect. A module lets these through; the +shared code logs, records and decides. + +**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page +handed to `rows`, its raw results as dicts with an `id` (counted, by distinct +id, to decide completeness), the total the host states or None, and the +cursor of the next page or None after the last. The first call gets cursor +None. A board with a total is complete when as many distinct ids were read; +one without, when the read ended on an empty page. A 429 after the first page +ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY` +apart. A paged module may also have `parse(body, slug)` for one page's body. + +**Rows.** A `Row` is a dict with `source_id` (the host's own id for the +role, unique on the board), `title`, `location`, `url` (http or https, +checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see +`_shared.parse_dt`), `department`, `description` (plain text, see +`desk.sources.strip_html`), `remote` (True, False, a word or None) and +`salary_text`; and, where the host states pay, `salary_min`, `salary_max` +(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`. + +**Detail.** `detail(get, slug, row)` reads one role's own page and returns a +dict of the fields it found. The shared code calls it for roles not yet read +(`ats:detailed:`), at most `ats.DETAIL_CAP` (200) a board a pull, +`ats.PAGE_DELAY` apart, the roles never tried before those that failed +(`ats:detailfailed:`). Only `ats.DETAIL_FIELDS` are taken: +`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` +and `posted`, the keys whose columns survive a later pull that sends the +listing's empty value; any other key is left out with a debug log line. +`title`, `url`, `department` and `salary_text` cannot be filled from +`detail`: the listing rewrites them every pull. A role not read in detail +on a pull is sent with an empty description, so the one read earlier stays. +A 404 or a 410 counts as read; any other failure is logged and retried on a +later pull; a 429 ends the detail reads of that board for the pull. +Completeness is decided by the listing alone. + +**robots.txt.** With `robots_page(slug)`, the shared code reads +`https:///robots.txt` through `get` before the listing, at most +once a day per board (`ats:robots:`), following a redirect to the +host's `www.` twin or back and no other. The board fails with "robots.txt +disallows it" when the rules for `*` or for the app's own token disallow the +page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a +final `$` in paths, case-sensitive paths). A 4xx other than 429 means no +rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails +the board with "robots.txt could not be read" and is asked again next pull. +`robots_page` must be on a host `allowed(slug)` admits. + +**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when +the slug is a board; `probe_ok(body)` narrows that when a host answers an +empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed +name is probed as, `(guess,)` without it. A host without `probe_url` is never +probed, only found by a link. +""" + +from __future__ import annotations + +import importlib +import pkgutil +import re +from collections.abc import Iterable +from types import ModuleType + +#: The rank of a module that states none: after the eight first hosts, whose +#: ranks (10 to 80) keep the order they had before there were modules. +DEFAULT_RANK = 100 + +#: What every host module defines. +REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url") + +#: The functions a module may define; each must be callable if present. +FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows", + "fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs") + + +def problems(module: ModuleType, name: str | None = None) -> list[str]: + """What `module` lacks or gets wrong against the contract, one phrase + each; empty for a module that keeps it. `name` is the module's own name + in the package, which `NAME` must equal.""" + out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)] + if name is not None and getattr(module, "NAME", name) != name: + out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}") + if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL): + out.append("LABEL is not a non-empty string") + if not isinstance(getattr(module, "KEEP_CASE", False), bool): + out.append("KEEP_CASE is not a bool") + rank = getattr(module, "RANK", DEFAULT_RANK) + if not isinstance(rank, int) or isinstance(rank, bool): + out.append("RANK is not an int") + patterns = getattr(module, "PATTERNS", ()) + if isinstance(patterns, str) or not isinstance(patterns, Iterable): + out.append("PATTERNS is not a tuple of strings") + else: + for pattern in patterns: + try: + re.compile(pattern) + except (TypeError, re.error) as exc: + out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})") + out += [f"{attr} is not callable" for attr in FUNCTIONS + if hasattr(module, attr) and not callable(getattr(module, attr))] + if hasattr(module, "page"): + if not hasattr(module, "rows"): + out.append("page without rows") + elif not (hasattr(module, "feed_url") and hasattr(module, "parse")): + out.append("neither feed_url and parse nor page and rows") + if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"): + out.append("probe_ok without probe_url") + if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"): + out.append("probe_slugs without probe_url") + return out + + +def load(path: Iterable[str], package: str) -> dict[str, ModuleType]: + """`NAME -> module` for every module of `package` (found on `path`) whose + name does not start with `_`, in order of `(RANK, NAME)`. Raises + ImportError naming the first module that breaks the contract.""" + modules: list[ModuleType] = [] + for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name): + if info.name.startswith("_"): + continue + module = importlib.import_module(f"{package}.{info.name}") + if found := problems(module, info.name): + raise ImportError(f"{package}.{info.name} breaks the host contract: " + + "; ".join(found)) + modules.append(module) + modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME)) + return {module.NAME: module for module in modules} + + +#: host -> its module, in order of `(RANK, NAME)`. +HOSTS: dict[str, ModuleType] = load(__path__, __name__) diff --git a/plugins/atshosts/_shared.py b/plugins/atshosts/_shared.py new file mode 100644 index 0000000..c31441f --- /dev/null +++ b/plugins/atshosts/_shared.py @@ -0,0 +1,192 @@ +"""What the host modules share: the row and page shapes, the fetch they are +handed, and the date, URL and salary helpers more than one host needs. + +Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both +import it without either importing the other. The leading underscore keeps +`plugins.atshosts` from listing it as a host. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime +from typing import Any, Protocol + +import httpx + +#: What every parser turns a feed into: one dict per row, with the keys +#: `source_id`, `title`, `location`, `url`, `posted`, `department`, +#: `description`, `remote` and `salary_text`, and, where a host states pay, +#: `salary_min`, `salary_max` and `currency`. +Row = dict[str, Any] + +#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded +#: page, its raw results (dicts with an `id`, counted for completeness), the +#: total it states (None for a host that states none), and the cursor of the +#: next page (None after the last). The first call gets cursor None. +Page = tuple[Any, list[dict[str, Any]], Any, Any] + + +class Get(Protocol): + """The fetch a host is handed: `ats.py`'s own, bound to one board, so the + allow-list, the redirect rule, the byte cap and the deadline apply to + every request a host makes. A POST sends `payload` as JSON; `headers` are + added to (and may replace) the default User-Agent and Accept. Returns the + body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see + "Fetching" in the package's docstring for the rest.""" + + def __call__(self, url: str, *, method: str = "GET", + payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None) -> bytes: ... + + +def is_web_url(url: str) -> bool: + """A trimmed http(s) URL, or refuse — a feed is external content and a + `javascript:` or `data:` link must never reach an href. Port of + openings.go's `webURL`.""" + url = (url or "").strip() + if not url: + return False + parsed = httpx.URL(url) + return parsed.scheme in ("http", "https") and bool(parsed.host) + + +def only(host: str) -> Callable[[str], bool]: + """The allow-list of a board that is fetched from one host name only, + which is every host so far. A host name has no case, and a slug that + names one (`Acme` of acme.recruitee.com) may be written with capitals.""" + want = host.lower() + return lambda h: h.lower() == want + + +#: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`). +#: A slug that is one URL path segment: what the hosts that put the slug in +#: the path accept, and nothing that could add a path, a query or a fragment. +_SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+") + +#: A slug that is one DNS label, for the hosts that put it in the host name. +_LABEL = re.compile(r"[A-Za-z0-9-]+") + + +def path_slug_ok(slug: str) -> bool: + return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..") + + +def label_slug_ok(slug: str) -> bool: + return bool(_LABEL.fullmatch(slug)) + + +def parse_dt(value: Any) -> datetime | None: + """An aware datetime from whatever a board sends: ISO-8601 with an + offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC", + or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now".""" + if value is None: + return None + if isinstance(value, int | float): + if value <= 0: + return None + seconds = value / 1000 if value > 10**12 else value + return datetime.fromtimestamp(seconds, tz=UTC) + text = str(value).strip() + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text) + if not match: + try: + parsed = parsedate_to_datetime(text) + except (TypeError, ValueError): + return None + return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) + parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00") + return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) + + +# ---------------------------------------------------------- structured pay +# +# A board's own salary range, mapped to yearly `salary_min`/`salary_max` +# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks +# `raw` into, per docs/decisions.md's "raw is transport, columns are +# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give +# structured pay on their list endpoint; Workable, SmartRecruiters and +# Teamtailor do not (checked live, 2026-09-29), so their rows leave the +# columns out and `ATSSource.fetch` reads a plain '' or None for them. + + +def num(value: Any) -> float | None: + """A positive number from whatever a board sends (int, float, or a + numeral string), or None for anything else, including zero and a + negative — a board's placeholder 0 (an unfilled field, not a real + figure) must not become a salary floor of zero.""" + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if number > 0 else None + + +def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any, + factors: dict[str, int]) -> tuple[int | None, int | None, str]: + """`(salary_min, salary_max, currency)` as yearly integers, or `(None, + None, "")` when there is no currency, no bound, or `interval` is not one + of `factors`' own keys — which is how an hourly or a daily rate is left + out rather than annualised into a number nobody stated. `factors` maps + the board's own interval spelling to the multiplier that makes it + yearly: 1 for a figure already yearly, 12 for a monthly one.""" + code = str(currency or "").strip().upper() + factor = factors.get(str(interval or "").strip()) + if not code or not factor: + return None, None, "" + lo, hi = num(min_value), num(max_value) + if lo is None and hi is None: + return None, None, "" + return (int(lo * factor) if lo is not None else None, + int(hi * factor) if hi is not None else None, code) + + +# ------------------------------------------------------------- salary_text +# +# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose: +# hourly and daily rates, a Greenhouse role with several pay zones, and a +# range with no currency are all dropped from those columns rather than +# guessed into a wrong yearly figure. `salary_text` is where that dropped +# information survives — one human-readable line per parser, written +# whether or not the pay conformed, so a person can read it and do their own +# sums and a stricter analysis can find the roles the columns left out +# (`Role.salary_state`, `app/models.py`). + +#: Formats a number the way a salary line writes it: no decimals for a whole +#: figure (the normal case for a yearly or monthly one), two for anything +#: else (an hourly or daily rate quoted with cents). +def fmt_amount(value: float) -> str: + return f"{int(value):,}" if value == int(value) else f"{value:,.2f}" + + +def bounds_words(lo: float | None, hi: float | None) -> str: + """"70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one + bound stated, or "" with neither.""" + if lo is not None and hi is not None: + return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}" + if lo is not None: + return f"from {fmt_amount(lo)}" + if hi is not None: + return f"up to {fmt_amount(hi)}" + return "" + + +def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str: + """One statement of pay as a human-readable line — "EUR 70,000 to 90,000 + per year" — or "" when there is no bound to report at all. A currency + that was not stated does not drop the line, unlike the normalised + columns: it is said plainly instead, "80,000 to 100,000 per year (no + currency stated)", so the figure is not lost along with the currency.""" + bounds = bounds_words(lo, hi) + if not bounds: + return "" + tail = f" per {period}" if period else "" + code = str(currency or "").strip().upper() + return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)" diff --git a/plugins/atshosts/ashby.py b/plugins/atshosts/ashby.py new file mode 100644 index 0000000..0513477 --- /dev/null +++ b/plugins/atshosts/ashby.py @@ -0,0 +1,122 @@ +"""Ashby: `ashby:`, the whole board in one JSON answer. A +board name may carry a dot (`jimdo.com`) or be a uuid. + +The field map is ported from `openings.go` in +[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT). +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount + +NAME = "ashby" +LABEL = "Ashby" +RANK = 30 +PATTERNS = ( + r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)", + r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)", +) + +_API = "api.ashbyhq.com" + + +def slug_ok(slug: str) -> bool: + return path_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_API) + + +def careers_url(slug: str) -> str: + return f"https://jobs.ashbyhq.com/{slug}" + + +def feed_url(slug: str) -> str: + return f"https://{_API}/posting-api/job-board/{slug}?includeCompensation=true" + + +def probe_url(slug: str) -> str: + return f"https://{_API}/posting-api/job-board/{slug}" + + +def probe_ok(body: bytes) -> bool: + """Ashby answers an empty listing for an unknown name, so a listing + without jobs counts as no board.""" + return b'"jobs":[' in body and b'"jobs":[]' not in body + + +def _salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: + """The yearly range from `compensation.summaryComponents` (needs + `?includeCompensation=true` on the feed URL, see `feed_url`): the first + `Salary` component, yearly or monthly (a monthly one multiplied by 12); + an hourly one is left out rather than annualised.""" + for component in components: + if component.get("compensationType") != "Salary": + continue + return yearly_amount(component.get("minValue"), component.get("maxValue"), + component.get("currencyCode"), component.get("interval"), + {"1 YEAR": 1, "1 MONTH": 12}) + return None, None, "" + + +def _salary_text(compensation: dict[str, Any]) -> str: + """One line per `summaryComponents` entry that states a bound, labelled + by the board's own `compensationType` when more than one does (a salary + range beside a bonus range, say); with just the one, the label is left + off, the same as Greenhouse's. `interval` is " " ("1 YEAR", + "1 HOUR"); the last word, lowered, is the period. With nothing to build + a line from, Ashby's own prose summary (`compensationTierSummary`, else + `scrapeableCompensationSalarySummary`) is what is left to show.""" + components = compensation.get("summaryComponents") or [] + stated = [c for c in components if num(c.get("minValue")) is not None + or num(c.get("maxValue")) is not None] + lines = [] + for component in stated: + period = str(component.get("interval") or "").split(" ")[-1].lower() + line = pay_line(num(component.get("minValue")), num(component.get("maxValue")), + component.get("currencyCode"), period) + if not line: + continue + if len(stated) > 1: + label = str(component.get("compensationType") or "").strip() + line = f"{label}: {line}" if label else line + lines.append(line) + if lines: + return "; ".join(lines) + return str(compensation.get("compensationTierSummary") + or compensation.get("scrapeableCompensationSalarySummary") or "") + + +def parse(body: bytes, slug: str) -> list[Row]: + data = json.loads(body) + rows = [] + for j in data.get("jobs") or []: + title = (j.get("title") or "").strip() + url = (j.get("jobUrl") or j.get("applyUrl") or "").strip() + if not title or not is_web_url(url): + continue + description = j.get("descriptionHtml") or j.get("descriptionPlain") or "" + compensation = j.get("compensation") or {} + salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or []) + rows.append({ + "source_id": str(j.get("id") or ""), + "title": title, + "location": (j.get("location") or "").strip(), + "url": url, + "posted": parse_dt(j.get("publishedAt")), + "department": (j.get("department") or j.get("team") or "").strip(), + "description": strip_html(description), + "remote": j.get("isRemote"), + "salary_text": _salary_text(compensation), + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/greenhouse.py b/plugins/atshosts/greenhouse.py new file mode 100644 index 0000000..ce0ceb6 --- /dev/null +++ b/plugins/atshosts/greenhouse.py @@ -0,0 +1,139 @@ +"""Greenhouse: `greenhouse:`, the whole board in one JSON answer. + +The field map is ported from `openings.go` in +[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT). +""" + +from __future__ import annotations + +import html +import json +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line + +NAME = "greenhouse" +LABEL = "Greenhouse" +RANK = 10 +PATTERNS = ( + r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)", + r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)", +) + +_API = "boards-api.greenhouse.io" + + +def slug_ok(slug: str) -> bool: + return path_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_API) + + +def careers_url(slug: str) -> str: + return f"https://job-boards.greenhouse.io/{slug}" + + +def feed_url(slug: str) -> str: + return f"https://{_API}/v1/boards/{slug}/jobs?content=true&pay_transparency=true" + + +def fallback_url(slug: str) -> str: + """The feed without `content=true`, read once when the full one exceeds + the byte cap: the descriptions are lost, but the titles land rather than + the whole board being skipped.""" + return f"https://{_API}/v1/boards/{slug}/jobs" + + +def probe_url(slug: str) -> str: + return f"https://{_API}/v1/boards/{slug}" + + +def _salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: + """The yearly range from `pay_input_ranges` (needs `pay_transparency=true` + on the feed URL, see `feed_url`), which the list endpoint already carries + in the same request — no per-role fetch. Only when there is exactly one + range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations", + one per office) gives no way to tell which office the role in hand is + at, and picking one would misrepresent the columns. Greenhouse's own + ranges are cents, and yearly by convention except an hourly or a daily + one, named as such in `title` with no separate field to check — those + are skipped rather than annualised into a number nobody stated.""" + if len(ranges) != 1: + return None, None, "" + entry = ranges[0] + title = str(entry.get("title") or "").lower() + if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")): + return None, None, "" + code = str(entry.get("currency_type") or "").strip().upper() + if not code: + return None, None, "" + lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents")) + if lo is None and hi is None: + return None, None, "" + return (int(lo / 100) if lo is not None else None, + int(hi / 100) if hi is not None else None, code) + + +def _salary_text(ranges: list[dict[str, Any]]) -> str: + """One line per `pay_input_ranges` entry, joined with "; " — every range + Greenhouse states, not only the single one `_salary` accepts for the + columns. With more than one, each is labelled with the board's own + `title` ("Zone 1 Pay Range: ..."), since that is the only way to tell the + ranges apart; with just the one, the label adds nothing and is left off. + The hour/day check is the same one `_salary` uses to keep those out of + the yearly columns; here it only picks the word that follows "per".""" + multi = len(ranges) > 1 + lines = [] + for entry in ranges: + title = str(entry.get("title") or "").strip() + low_title = title.lower() + period = ("hour" if any(w in low_title for w in ("hour", "/hr")) + else "day" if "day" in low_title else "year") + lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents")) + line = pay_line(lo / 100 if lo is not None else None, + hi / 100 if hi is not None else None, + entry.get("currency_type"), period) + if not line: + continue + if multi: + label = title.rstrip(":").strip() or "Range" + line = f"{label}: {line}" + lines.append(line) + return "; ".join(lines) + + +def parse(body: bytes, slug: str) -> list[Row]: + data = json.loads(body) + rows = [] + for j in data.get("jobs") or []: + title = (j.get("title") or "").strip() + url = (j.get("absolute_url") or "").strip() + if not title or not is_web_url(url): + continue + departments = j.get("departments") or [] + # Greenhouse ships the description HTML-entity-escaped inside the + # JSON string (`<p>`), so it needs unescaping before the tags + # themselves can be stripped. + content = html.unescape(j.get("content") or "") + ranges = j.get("pay_input_ranges") or [] + salary_min, salary_max, currency = _salary(ranges) + rows.append({ + "source_id": str(j.get("id") or ""), + "title": title, + "location": ((j.get("location") or {}).get("name") or "").strip(), + "url": url, + "posted": parse_dt(j.get("first_published") or j.get("updated_at")), + "department": (departments[0].get("name") if departments else "") or "", + "description": strip_html(content), + "remote": None, # Greenhouse's feed does not say + "salary_text": _salary_text(ranges), + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/lever.py b/plugins/atshosts/lever.py new file mode 100644 index 0000000..f988ce6 --- /dev/null +++ b/plugins/atshosts/lever.py @@ -0,0 +1,121 @@ +"""Lever: `lever:`, or `lever:eu/` for a board on Lever's EU +instance, the whole board in one JSON answer. + +Lever runs two instances that share nothing but the API: `api.lever.co` and +`api.eu.lever.co`, with the pages on `jobs.lever.co` and `jobs.eu.lever.co`. +A site exists on one of them only (TomTom and Prima answer 404 on the other, +2026-09-29), and the same name on both would be two companies, so the region +is part of the slug. It is a prefix a plain site name cannot carry, since a +Lever site name has no slash, which leaves every `lever:` target as it +was. + +The field map is ported from `openings.go` in +[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT). +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount + +NAME = "lever" +LABEL = "Lever" +RANK = 20 +#: Two groups on the EU hosts, `eu` and the site, which discovery joins into +#: the slug `eu/`. +PATTERNS = ( + r"jobs\.lever\.co/([A-Za-z0-9_-]+)", + r"api\.lever\.co/v0/postings/([A-Za-z0-9_-]+)", + r"jobs\.(eu)\.lever\.co/([A-Za-z0-9_-]+)", + r"api\.(eu)\.lever\.co/v0/postings/([A-Za-z0-9_-]+)", +) + +#: The region prefixes a slug may carry, to the infix of both host names. +_REGIONS = {"eu": "eu."} + + +def _split(slug: str) -> tuple[str, str]: + """`(host infix, site)`: `("eu.", "tomtom")` for `eu/tomtom`, `("", "acme")` + for `acme`.""" + region, sep, site = slug.partition("/") + return (_REGIONS.get(region, "?"), site) if sep else ("", slug) + + +def slug_ok(slug: str) -> bool: + infix, site = _split(slug) + return infix != "?" and path_slug_ok(site) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(f"api.{_split(slug)[0]}lever.co") + + +def careers_url(slug: str) -> str: + infix, site = _split(slug) + return f"https://jobs.{infix}lever.co/{site}" + + +def feed_url(slug: str) -> str: + infix, site = _split(slug) + return f"https://api.{infix}lever.co/v0/postings/{site}?mode=json" + + +def probe_url(slug: str) -> str: + infix, site = _split(slug) + return f"https://api.{infix}lever.co/v0/postings/{site}?limit=1&mode=json" + + +def probe_slugs(guess: str) -> tuple[str, ...]: + """A name is tried on both instances, the US one first, since a site that + names neither leaves no other way to tell which one it is on.""" + return (guess, *(f"{region}/{guess}" for region in _REGIONS)) + + +def _salary_text(salary_range: dict[str, Any]) -> str: + """One line from `salaryRange`, whatever its `interval` — including an + hourly or daily one, which `yearly_amount` drops from the columns. + Lever's interval is "per--" ("per-year-salary", + "per-hour-wage"); the middle word is the period.""" + interval = str(salary_range.get("interval") or "") + parts = interval.split("-") + period = parts[1] if len(parts) >= 3 else "" + return pay_line(num(salary_range.get("min")), num(salary_range.get("max")), + salary_range.get("currency"), period) + + +def parse(body: bytes, slug: str) -> list[Row]: + data = json.loads(body) + if not isinstance(data, list): + return [] + rows = [] + for j in data: + title = (j.get("text") or "").strip() + url = (j.get("hostedUrl") or "").strip() + if not title or not is_web_url(url): + continue + categories = j.get("categories") or {} + description = j.get("description") or j.get("descriptionPlain") or "" + salary_range = j.get("salaryRange") or {} + salary_min, salary_max, currency = yearly_amount( + salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"), + salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12}) + rows.append({ + "source_id": str(j.get("id") or ""), + "title": title, + "location": (categories.get("location") or "").strip(), + "url": url, + "posted": parse_dt(j.get("createdAt")), + "department": (categories.get("department") or categories.get("team") or "").strip(), + "description": strip_html(description), + "remote": (j.get("workplaceType") or "").strip() or None, + "salary_text": _salary_text(salary_range), + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/personio.py b/plugins/atshosts/personio.py new file mode 100644 index 0000000..0e1116c --- /dev/null +++ b/plugins/atshosts/personio.py @@ -0,0 +1,101 @@ +"""Personio: `personio:`, the whole board in one XML answer from +the company's own subdomain of jobs.personio.de. + +Written from one real board on 2026-09-23; jobhunt-hq did not read +Personio. +""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Callable + +from desk.sources import strip_html + +from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount + +NAME = "personio" +LABEL = "Personio" +RANK = 70 +PATTERNS = (r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)",) + +#: Personio's `salaryInformation/type` spellings, to the plain period word +#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through. +_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour", + "daily": "day", "weekly": "week"} + + +def slug_ok(slug: str) -> bool: + return label_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + """The company's own subdomain, so one company's slug cannot pass for + another's.""" + return only(f"{slug}.jobs.personio.de") + + +def careers_url(slug: str) -> str: + return f"https://{slug}.jobs.personio.de" + + +def feed_url(slug: str) -> str: + return f"https://{slug}.jobs.personio.de/xml" + + +def probe_url(slug: str) -> str: + return feed_url(slug) + + +def probe_ok(body: bytes) -> bool: + return b"" in body + + +def _salary_text(salary_info: ET.Element) -> str: + """One line from a `` element, whatever its `type`, + including an hourly or daily one that `yearly_amount` drops from the + columns.""" + period_raw = (salary_info.findtext("type") or "").strip().lower() + period = _PERIODS.get(period_raw, period_raw) + return pay_line(num(salary_info.findtext("min")), num(salary_info.findtext("max")), + salary_info.findtext("currencyCode"), period) + + +def parse(body: bytes, slug: str) -> list[Row]: + """`{slug}.jobs.personio.de/xml` -> `...`. No job + URL in the feed itself; Personio's own page is `/job/{id}`, confirmed + live against a real posting (2026-09-23).""" + root = ET.fromstring(body) + rows = [] + for position in root.findall("position"): + job_id = (position.findtext("id") or "").strip() + title = (position.findtext("name") or "").strip() + if not job_id or not title: + continue + description = " ".join( + (value.text or "") for value in position.findall("jobDescriptions/jobDescription/value")) + salary_info = position.find("salaryInformation") + salary_min = salary_max = None + currency = "" + salary_text = "" + if salary_info is not None: + salary_min, salary_max, currency = yearly_amount( + salary_info.findtext("min"), salary_info.findtext("max"), + salary_info.findtext("currencyCode"), salary_info.findtext("type"), + {"yearly": 1, "monthly": 12}) + salary_text = _salary_text(salary_info) + rows.append({ + "source_id": job_id, + "title": title, + "location": (position.findtext("office") or "").strip(), + "url": f"https://{slug}.jobs.personio.de/job/{job_id}", + "posted": parse_dt(position.findtext("createdAt")), + "department": (position.findtext("department") or "").strip(), + "description": strip_html(description), + "remote": "remote" in (position.findtext("office") or "").lower() or None, + "salary_text": salary_text, + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/recruitee.py b/plugins/atshosts/recruitee.py new file mode 100644 index 0000000..81e9903 --- /dev/null +++ b/plugins/atshosts/recruitee.py @@ -0,0 +1,84 @@ +"""Recruitee: `recruitee:`, the whole board in one JSON answer +from the company's own subdomain of recruitee.com. + +Written from one real board on 2026-09-23; jobhunt-hq did not read +Recruitee. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount + +NAME = "recruitee" +LABEL = "Recruitee" +RANK = 60 +PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",) + + +def slug_ok(slug: str) -> bool: + return label_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + """The company's own subdomain, so one company's slug cannot pass for + another's.""" + return only(f"{slug}.recruitee.com") + + +def careers_url(slug: str) -> str: + return f"https://{slug}.recruitee.com" + + +def feed_url(slug: str) -> str: + return f"https://{slug}.recruitee.com/api/offers/" + + +def probe_url(slug: str) -> str: + return feed_url(slug) + + +def probe_ok(body: bytes) -> bool: + return b'"offers":[' in body and b'"offers":[]' not in body + + +def _salary_text(salary: dict[str, Any]) -> str: + """One line from Recruitee's `salary` object; `period` is already a + plain word ("year", "month").""" + period = str(salary.get("period") or "").strip().lower() + return pay_line(num(salary.get("min")), num(salary.get("max")), + salary.get("currency"), period) + + +def parse(body: bytes, slug: str) -> list[Row]: + data = json.loads(body) + rows = [] + for j in data.get("offers") or []: + title = (j.get("title") or "").strip() + url = (j.get("careers_url") or "").strip() + if not title or not is_web_url(url): + continue + salary = j.get("salary") if isinstance(j.get("salary"), dict) else {} + salary_min, salary_max, currency = yearly_amount( + salary.get("min"), salary.get("max"), salary.get("currency"), + salary.get("period"), {"year": 1, "month": 12}) + rows.append({ + "source_id": str(j.get("id") or ""), + "title": title, + "location": (j.get("location") or j.get("city") or "").strip(), + "url": url, + "posted": parse_dt(j.get("published_at") or j.get("created_at")), + "department": (j.get("department") or "").strip(), + "description": strip_html(j.get("description") or ""), + "remote": j.get("remote"), + "salary_text": _salary_text(salary), + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/smartrecruiters.py b/plugins/atshosts/smartrecruiters.py new file mode 100644 index 0000000..62d2633 --- /dev/null +++ b/plugins/atshosts/smartrecruiters.py @@ -0,0 +1,90 @@ +"""SmartRecruiters: `smartrecruiters:`, paged by offset +with a stated total. The identifier is case-sensitive (`BoschGroup`). + +Written from one real board on 2026-09-23; jobhunt-hq did not read +SmartRecruiters. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok + +NAME = "smartrecruiters" +LABEL = "SmartRecruiters" +RANK = 50 +PATTERNS = ( + r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)", +) + +_API = "api.smartrecruiters.com" + +#: The page size asked for, which is also the most the host serves per page. +PAGE_SIZE = 100 + + +def slug_ok(slug: str) -> bool: + return path_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_API) + + +def careers_url(slug: str) -> str: + return f"https://jobs.smartrecruiters.com/{slug}" + + +def probe_url(slug: str) -> str: + return f"https://{_API}/v1/companies/{slug}/postings?limit=1" + + +def probe_ok(body: bytes) -> bool: + """An unknown identifier answers an empty listing, so a listing without + postings counts as no board.""" + return b'"totalFound":' in body and b'"totalFound":0' not in body + + +def page(get: Get, slug: str, cursor: Any) -> Page: + offset = int(cursor or 0) + data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings" + f"?offset={offset}&limit={PAGE_SIZE}")) + content = data.get("content") or [] + return data, content, data.get("totalFound"), offset + len(content) if content else None + + +def parse(body: bytes, slug: str) -> list[Row]: + return rows(json.loads(body), slug) + + +def rows(data: dict[str, Any], slug: str) -> list[Row]: + """One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings + -> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a + page, whatever `limit` asks for; the next is `?offset=` further on. The + list omits the job description (only the per-posting detail endpoint has + it, and fetching one per row would be hundreds of extra requests per + board), so `description` stays empty here.""" + out = [] + for j in data.get("content") or []: + title = (j.get("name") or "").strip() + job_id = str(j.get("id") or "") + if not title or not job_id: + continue + location = j.get("location") or {} + loc_text = (location.get("fullLocation") + or ", ".join(p for p in (location.get("city"), location.get("country")) if p)) + out.append({ + "source_id": job_id, + "title": title, + "location": (loc_text or "").strip(), + "url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}", + "posted": parse_dt(j.get("releasedDate")), + "department": ((j.get("department") or {}).get("label") or "").strip(), + "description": "", + "remote": location.get("remote"), + "salary_text": "", # SmartRecruiters' list endpoint states no pay at all + }) + return out diff --git a/plugins/atshosts/teamtailor.py b/plugins/atshosts/teamtailor.py new file mode 100644 index 0000000..0f0481e --- /dev/null +++ b/plugins/atshosts/teamtailor.py @@ -0,0 +1,137 @@ +"""Teamtailor: `teamtailor:` for acme.teamtailor.com, or, for a +career site on the company's own domain, that whole host name: +`teamtailor:careers.acme.com`. An RSS feed paged by offset that states no +total. + +Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor. +""" + +from __future__ import annotations + +import html +import re +import xml.etree.ElementTree as ET +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Get, Page, Row, is_web_url, only, parse_dt + +NAME = "teamtailor" +LABEL = "Teamtailor" +RANK = 80 +PATTERNS = ( + r"https?://([a-z0-9-]+)\.teamtailor\.com", + # No slug: a career site on the company's own domain, which only loads + # Teamtailor's CDN. Discovery reads its board off the page's host. + r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets", +) + +#: The page size, asked for as `per_page`. A feed with neither parameter +#: answers with every role (73 of 73 at Shine, 2026-09-29), but says nowhere +#: that it is the whole list, so this reads by page to an empty one; +#: `per_page=200` was answered in full too, and the cursor advances by what a +#: page returned, so a smaller cap on the host's side would not skip a role. +PAGE_SIZE = 50 + +#: Only a DNS name with a letter in its last label is a site, so an address +#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could +#: be pointed at; `ats._get` still refuses every host but this one, redirects +#: included. +_HOST = re.compile(r"[a-z0-9-]+(\.[a-z0-9-]+)*") + +#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`. +_TT = "{https://teamtailor.com/locations}" + +#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing +#: about where the role is worked and `temporary` is remote for a spell only, +#: so both stay unknown rather than become "onsite" or "remote". +_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"} + + +def site(slug: str) -> str: + """The host a career site is served from: `acme` is acme.teamtailor.com, + and a slug with a dot is a site on the company's own domain, + `careers.acme.com`, taken whole (the feed path is the same on both).""" + slug = slug.strip().lower() + return slug if "." in slug else f"{slug}.teamtailor.com" + + +def slug_ok(slug: str) -> bool: + host = site(slug) + return bool(slug) and slug == slug.strip() and bool(_HOST.fullmatch(host)) \ + and not host.rsplit(".", 1)[-1].isdigit() + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(site(slug)) + + +def careers_url(slug: str) -> str: + return f"https://{site(slug)}/jobs" + + +def probe_url(slug: str) -> str: + return f"https://{site(slug)}/jobs.rss" + + +def probe_ok(body: bytes) -> bool: + return b"" in body + + +def page(get: Get, slug: str, cursor: Any) -> Page: + offset = int(cursor or 0) + items = _items(get(f"https://{site(slug)}/jobs.rss?per_page={PAGE_SIZE}&offset={offset}")) + results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items] + return items, results, None, offset + len(items) if items else None + + +def parse(body: bytes, slug: str) -> list[Row]: + return rows(_items(body), slug) + + +def _items(body: bytes) -> list[ET.Element]: + """`{site}/jobs.rss` -> `...`, the feed every Teamtailor + career site serves (checked live on a dozen sites, 2026-09-29). A missing + `channel` is a page that is not a feed, and raises.""" + channel = ET.fromstring(body).find("channel") + if channel is None: + raise ValueError("not a Teamtailor feed: no channel") + return channel.findall("item") + + +def rows(items: list[ET.Element], slug: str) -> list[Row]: + """One page of items. The item has the title, the job URL (on the site's + own domain), a `guid` uuid, `pubDate`, the HTML `description` in full, + `remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the + `tt:` namespace, `locations` (a list: `city` and `country` each, `name` free + text), `department` and `role`, either of which may be empty. `place` joins + the distinct locations with "; ", which `country_of` reads a location at a + time. No pay is stated anywhere in the feed.""" + out = [] + for item in items: + title = (item.findtext("title") or "").strip() + url = (item.findtext("link") or "").strip() + if not title or not is_web_url(url): + continue + places: list[str] = [] + for loc in item.findall(f"{_TT}locations/{_TT}location"): + city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip() + country = (loc.findtext(f"{_TT}country") or "").strip() + place = city if country == city else ", ".join(p for p in (city, country) if p) + if place and place not in places: + places.append(place) + out.append({ + "source_id": (item.findtext("guid") or url).strip(), + "title": title, + "location": "; ".join(places), + "url": url, + "posted": parse_dt(item.findtext("pubDate")), + "department": ((item.findtext(f"{_TT}department") or "").strip() + or (item.findtext(f"{_TT}role") or "").strip()), + "description": html.unescape(strip_html(item.findtext("description") or "")), + "remote": _REMOTE.get((item.findtext("remoteStatus") or "").strip()), + "salary_text": "", # Teamtailor's feed states no pay at all + }) + return out diff --git a/plugins/atshosts/workable.py b/plugins/atshosts/workable.py new file mode 100644 index 0000000..9495065 --- /dev/null +++ b/plugins/atshosts/workable.py @@ -0,0 +1,85 @@ +"""Workable: `workable:`, paged ten roles at a time with a stated +total. + +Written from one real board, Hugging Face's, on 2026-09-28, when the first +board with roles was seen; jobhunt-hq did not read Workable. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok + +NAME = "workable" +LABEL = "Workable" +RANK = 40 +PATTERNS = ( + r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)", + r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com", +) + +_API = "apply.workable.com" + +# No probe: Workable's API sits behind a rate limit that a run of guesses +# trips, and a Workable site links its board plainly anyway. + + +def slug_ok(slug: str) -> bool: + return path_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_API) + + +def careers_url(slug: str) -> str: + return f"https://{_API}/{slug}" + + +def page(get: Get, slug: str, cursor: Any) -> Page: + """One POST of the list; the next page is the same POST with + `{"token": nextPage}`, and the last page has no `nextPage`.""" + url = f"https://{_API}/api/v3/accounts/{slug}/jobs" + data = json.loads(get(url, method="POST", payload={"token": cursor} if cursor else {})) + return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None + + +def parse(body: bytes, slug: str) -> list[Row]: + return rows(json.loads(body), slug) + + +def rows(data: dict[str, Any], slug: str) -> list[Row]: + """One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs + -> `{"total": int, "results": [...], "nextPage": token}`, as Hugging + Face's board answered on 2026-09-28. A result carries no URL and no text: + the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and + the description is only on the per-job endpoint, one request per role, + so `description` stays empty here as it does for SmartRecruiters. + `location` is an object and `department` a list; `remote` is a flag + beside `workplace` (remote, hybrid, on_site).""" + out = [] + for j in data.get("results") or []: + title = (j.get("title") or "").strip() + shortcode = (j.get("shortcode") or "").strip() + if not title or not shortcode: + continue + loc = j.get("location") or {} + location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p) + dept = j.get("department") or "" + if isinstance(dept, list): + dept = ", ".join(d for d in dept if d) + out.append({ + "source_id": str(j.get("id") or shortcode), + "title": title, + "location": location, + "url": f"https://{_API}/{slug}/j/{shortcode}/", + "posted": parse_dt(j.get("published")), + "department": dept.strip(), + "description": "", + "remote": bool(j.get("remote")) or j.get("workplace") == "remote", + "salary_text": "", # Workable's list endpoint states no pay at all + }) + return out diff --git a/tests/test_ats.py b/tests/test_ats.py index 84adb4b..1a9ae27 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -19,6 +19,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats # noqa: E402 +from plugins.atshosts import smartrecruiters # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -384,7 +385,7 @@ def test_is_known_target_accepts_the_eight_hosts_and_refuses_others() -> None: "recruitee", "personio", "teamtailor"): assert ats.is_known_target(f"{host}:acme") assert not ats.is_known_target("linkedin:acme") - assert not ats.is_known_target("workday:acme") + assert not ats.is_known_target("taleo:acme") assert not ats.is_known_target("greenhouse:") assert not ats.is_known_target("nohostnoseparator") # a Teamtailor slug names a host: its own subdomain, or a whole domain @@ -420,7 +421,7 @@ def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None: assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com" assert ats.careers_url("teamtailor:acme") == "https://acme.teamtailor.com/jobs" assert ats.careers_url("teamtailor:careers.acme.com") == "https://careers.acme.com/jobs" - assert ats.careers_url("workday:acme") == "" + assert ats.careers_url("taleo:acme") == "" # --------------------------------------------------------------- body cap @@ -570,7 +571,7 @@ def test_smartrecruiters_pages_by_offset_to_total_found( urls.append(url) return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json") - monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2) monkeypatch.setattr(ats, "_get", fake_get) rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] @@ -588,7 +589,7 @@ def test_a_board_that_ends_short_of_its_total_is_incomplete( def fake_get(client, url, host, slug, *, method="GET", payload=None): return empty if "offset=2" in url else _page("smartrecruiters.json") - monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2) monkeypatch.setattr(ats, "_get", fake_get) rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] assert (len(rows), complete) == (2, False) @@ -604,7 +605,7 @@ def test_a_repeated_row_does_not_count_twice_toward_the_total( page["content"] = page["content"][1:] return json.dumps(page).encode() - monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2) monkeypatch.setattr(ats, "_get", fake_get) rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] assert (len(rows), complete) == (2, False) diff --git a/tests/test_atshosts.py b/tests/test_atshosts.py new file mode 100644 index 0000000..4d9e1ee --- /dev/null +++ b/tests/test_atshosts.py @@ -0,0 +1,761 @@ +"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for +every host: the registry and its contract check, the fetch a host is handed, +the per-role detail reads, robots.txt, and Lever's EU instance. No network — +a fake host stands in where a behaviour belongs to no real host yet.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import types +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import ats, atshosts # noqa: E402 +from plugins.atshosts import lever # noqa: E402 + +FIXTURES = ROOT / "tests" / "fixtures" / "ats" + + +# --------------------------------------------------------------- registry + +def test_the_registry_finds_every_module() -> None: + names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py") + if not p.stem.startswith("_")) + assert sorted(atshosts.HOSTS) == names + assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters", + "recruitee", "personio", "teamtailor"} + assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items()) + # The Sources page's select and the plugin's description follow the modules. + assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS + assert "Teamtailor" in ats.ATSSource.description + + +def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str, + modules: dict[str, str]) -> list[str]: + package = tmp_path / name + package.mkdir() + (package / "__init__.py").write_text("") + for module, source in modules.items(): + (package / f"{module}.py").write_text(source) + monkeypatch.syspath_prepend(str(tmp_path)) + return [str(package)] + + +_GOOD = """ +NAME = "{name}" +LABEL = "Good" +PATTERNS = (r"good\\.example/([a-z]+)",) +def slug_ok(slug): return True +def allowed(slug): return lambda h: h == "good.example" +def careers_url(slug): return "https://good.example/" + slug +def feed_url(slug): return "https://good.example/" + slug + ".json" +def parse(body, slug): return [] +""" + + +def test_the_registry_loads_ats_by_path_without_the_repo_root_on_the_path( + tmp_path: Path) -> None: + """As `desk.sources.registry` does in a process the app did not set up: + another working directory, and `plugins` not importable at first.""" + code = ( + "import importlib.util, sys\n" + "assert importlib.util.find_spec('plugins') is None, 'plugins is importable'\n" + "from desk.sources import registry\n" + f"print(sorted(registry.available([{str(ROOT / 'plugins')!r}])))\n") + env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} + done = subprocess.run([sys.executable, "-c", code], cwd=tmp_path, env=env, + capture_output=True, text=True, timeout=60, check=False) + assert done.returncode == 0, done.stderr + assert "'ats'" in done.stdout and "'search'" in done.stdout + + +def test_the_registry_loads_a_package_by_name_and_skips_private_modules( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + path = _package(tmp_path, monkeypatch, "hostsok", { + "zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"), + "_helpers": "X = 1\n"}) + assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"] + + +def test_the_registry_orders_by_rank_then_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + path = _package(tmp_path, monkeypatch, "hostsranked", { + "alpha": _GOOD.format(name="alpha"), # the default, 100 + "beta": _GOOD.format(name="beta") + "RANK = 5\n", + "gamma": _GOOD.format(name="gamma") + "RANK = 100\n"}) + assert list(atshosts.load(path, "hostsranked")) == ["beta", "alpha", "gamma"] + + +def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None: + """The Sources page's select opens on Greenhouse, which its hint names, + and a guessed slug on two hosts goes to the one probed first, as before.""" + first = ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", + "personio", "teamtailor"] + assert list(ats.HOSTS)[:8] == first + assert ats.ATSSource.description.startswith( + "Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, " + "Recruitee, Personio, Teamtailor") + assert all(module.RANK < atshosts.DEFAULT_RANK + for name, module in atshosts.HOSTS.items() if name in first) + + +def test_the_registry_refuses_a_module_missing_a_required_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url") + path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken}) + with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"): + atshosts.load(path, "hostsbad") + + +def test_the_contract_check_names_each_problem() -> None: + module: Any = types.ModuleType("odd") + module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",) + module.slug_ok = module.allowed = module.careers_url = lambda slug: "" + module.page = lambda get, slug, cursor: None + module.probe_ok = lambda body: True + module.RANK, module.KEEP_CASE = "1", "yes" + found = atshosts.problems(module, "odd") + assert "NAME is 'other', not the module's name 'odd'" in found + assert "LABEL is not a non-empty string" in found + assert any(p.startswith("PATTERNS has '('") for p in found) + assert "page without rows" in found + assert "probe_ok without probe_url" in found + assert "RANK is not an int" in found and "KEEP_CASE is not a bool" in found + + +@pytest.mark.parametrize("host", ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", + "recruitee", "personio", "teamtailor"]) +def test_a_slug_is_matched_whole(host: str) -> None: + assert ats.is_known_target(f"{host}:acme") + for slug in ("acme\n", "acme\r\n", " acme", "acme ", "ac me", "acme/x", "acme?x", "acme#x"): + assert not ats.is_known_target(f"{host}:{slug}"), slug + + +def test_a_slug_in_the_host_name_is_allowed_whatever_its_case() -> None: + for host, api in (("recruitee", "acme.recruitee.com"), ("personio", "acme.jobs.personio.de")): + allow = ats._allowed_host(host, "Acme") + assert allow(api) and allow(api.upper()) + assert not allow(api.replace("acme", "other")) + + +# ------------------------------------------------------------ the fetch + +def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def test_get_sends_extra_headers_on_top_of_its_own() -> None: + seen: list[httpx.Headers] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.headers) + return httpx.Response(200, content=b"ok") + + with _client(handler) as client: + get = ats._getter(client, "lever", "acme") + assert get("https://api.lever.co/v0/postings/acme", + headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok" + get("https://api.lever.co/v0/postings/acme") + assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme" + assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT + assert seen[1]["accept"] == "application/json, text/xml" + assert "x-board" not in seen[1] + + +@pytest.mark.parametrize("site", [ + "metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa", + "app.localhost", "foo.0x7f000001", "printer.local.", +]) +def test_get_refuses_a_local_name_before_any_request(site: str) -> None: + """A Teamtailor slug may be any host name, so the allow-list admits + these; `_get` refuses them before a request is made.""" + asked: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + asked.append(str(request.url)) + return httpx.Response(200, content=b"") + + with _client(handler) as client: + # A trailing dot is no Teamtailor slug at all; the rest are, and are refused. + with pytest.raises(ValueError if site.endswith(".") else ats._Refused): + ats._fetch_rows(client, f"teamtailor:{site}") + for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/", + "https://2130706433/", "https://127.1/", "https://localhost/"): + with pytest.raises(ats._Refused): + ats._get(client, url, "teamtailor", site) + assert asked == [] + + +@pytest.mark.parametrize("location", [ + "https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x", + "https://localhost/x", +]) +def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(302, headers={"location": location}) + + with _client(handler) as client, pytest.raises(ats._Refused): + ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme") + + +def test_get_takes_port_443_written_out_and_refuses_another() -> None: + with _client(lambda request: httpx.Response(200, content=b"[]")) as client: + assert ats._get(client, "https://api.lever.co:443/v0/postings/acme", + "lever", "acme") == b"[]" + with pytest.raises(ats._Refused): + ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme") + + +def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None: + """Bound before the replacement, and still reaching it: `_get` is looked + up when a request is made.""" + client = object() + get = ats._getter(client, "lever", "acme") # type: ignore[arg-type] + calls: list[tuple[Any, ...]] = [] + + def replaced(*args: Any, **kwargs: Any) -> bytes: + calls.append((args, kwargs)) + return b"" + + monkeypatch.setattr(ats, "_get", replaced) + get("https://api.lever.co/v0/postings/acme", headers={"X": "1"}) + get("https://api.lever.co/v0/postings/acme", method="POST", payload={}) + assert calls == [ + ((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"), + {"method": "GET", "headers": {"X": "1"}}), + ((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"), + {"method": "POST", "payload": {}})] + + +# ------------------------------------------------------- a fake host + +def _fake_host(*, detail: Callable[..., Any] | None = None, + robots: bool = False) -> types.ModuleType: + """A host `fake` whose feed is a JSON list of ids, each role with a + teaser for a description, and, with `detail`, a per-role page; with + `robots`, its listing is the page robots.txt must allow.""" + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () + module.slug_ok = lambda slug: slug.isalnum() + module.allowed = lambda slug: lambda h: h == "jobs.fake.example" + module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" + module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list" + module.parse = lambda body, slug: [ + {"source_id": i, "title": f"Role {i}", "location": "", "department": "", + "url": f"https://jobs.fake.example/{slug}/{i}", "posted": None, + "description": "teaser", "remote": None, "salary_text": ""} + for i in json.loads(body)] + if detail is not None: + module.detail = detail + if robots: + module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list" + return module + + +def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError: + request = httpx.Request("GET", url) + return httpx.HTTPStatusError(str(code), request=request, + response=httpx.Response(code, request=request)) + + +@pytest.fixture +def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """`ats._get` answering from a dict of URL -> body or exception, and + remembering what it was asked. No delay between requests.""" + answers: dict[str, Any] = {} + asked: list[str] = [] + + def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, + www_twin=False): + asked.append(url) + if not ats._allowed_host(host, slug)(httpx.URL(url).host): + raise ats._Refused(url) + answer = answers.get(url, _status(404, url)) + if isinstance(answer, BaseException): + raise answer + return answer + + monkeypatch.setattr(ats, "_get", fake_get) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + return {"answers": answers, "asked": asked} + + +def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType, + state: dict[str, str] | None = None) -> tuple[list[Any], Any]: + monkeypatch.setitem(atshosts.HOSTS, "fake", module) + source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) + if state is not None: + source.state = state + return list(source.fetch()), source.state + + +def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: + """The fake host's detail: the role's own page is its description.""" + return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(), + "location": "Copenhagen"} + + +# ---------------------------------------------------------------- detail + +def test_detail_is_read_for_unread_roles_only( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full", + f"{base}/2": b"two in full", f"{base}/3": b"three in full"}) + module = _fake_host(detail=_detail) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])} + items, state = _run(monkeypatch, module, state) + + assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"] + assert [(i.source_id, i.body) for i in items] == [ + ("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")] + # What detail returns replaces the listing's field; a role not read keeps the listing's. + assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == "" + assert json.loads(state["ats:detailed:fake:acme"]) == [ + "fake:acme:1", "fake:acme:2", "fake:acme:3"] + assert state["ats:complete:fake:acme"] == "1" + + +def test_detail_reads_stop_at_the_cap_and_the_rest_wait( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + ids = [str(n) for n in range(5)] + web["answers"].update({f"{base}/list": json.dumps(ids).encode()} + | {f"{base}/{i}": f"full {i}".encode() for i in ids}) + monkeypatch.setattr(ats, "DETAIL_CAP", 2) + module = _fake_host(detail=_detail) + items, state = _run(monkeypatch, module) + assert [i.body for i in items] == ["full 0", "full 1", "", "", ""] + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"] + + web["asked"].clear() + items, state = _run(monkeypatch, module, state) + assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"] + assert [i.body for i in items] == ["", "", "full 2", "full 3", ""] + + +def test_a_detail_that_fails_skips_the_role_and_not_the_board( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one", + f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"}) + items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert [(i.source_id, i.body) for i in items] == [ + ("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")] + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"] + assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r" + assert state["ats:complete:fake:acme"] == "1" + + +def test_a_429_on_a_detail_stops_the_reads_for_the_pull( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one", + f"{base}/2": _status(429), f"{base}/3": b"three"}) + items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"] + assert [i.body for i in items] == ["one", "", ""] + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] + assert state["ats:complete:fake:acme"] == "1" + + +def test_ids_no_longer_listed_leave_the_detailed_list( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["2"]'}) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} + _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"] + + +def test_an_incomplete_listing_keeps_the_ids_it_did_not_list( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + """A listing cut short says nothing about the roles it left out, so their + ids stay, and the next pull does not read them in detail again.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["2"]'}) + monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: ( + atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False)) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} + _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] + assert state["ats:complete:fake:acme"] == "" + + +def test_fetch_board_reads_the_listing_only( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"}) + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) + rows, complete = ats.fetch_board("fake:acme") + assert ([r["description"] for r in rows], complete) == (["teaser"], True) + assert web["asked"] == [f"{base}/list"] + + +def test_a_host_without_detail_keeps_the_listings_description( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' + items, state = _run(monkeypatch, _fake_host()) + assert [i.body for i in items] == ["teaser"] + assert "ats:detailed:fake:acme" not in state + + +def test_a_detail_answering_404_or_410_counts_as_read( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410), + f"{base}/3": b"three"}) # 1 answers 404 + items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert [i.body for i in items] == ["", "", "three"] + assert json.loads(state["ats:detailed:fake:acme"]) == [ + "fake:acme:1", "fake:acme:2", "fake:acme:3"] + assert json.loads(state["ats:detailfailed:fake:acme"]) == [] + + +@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")]) +def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: + """Two roles that always fail and a cap of two: the third is read on the + second pull, since roles never tried go before those that failed.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure, + f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"}) + monkeypatch.setattr(ats, "DETAIL_CAP", 2) + module = _fake_host(detail=_detail) + _items, state = _run(monkeypatch, module) + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] + + web["asked"].clear() + items, state = _run(monkeypatch, module, state) + assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"] + assert [i.body for i in items] == ["", "", "three", "four"] + + # With nothing new left, the failed ones are tried again, within the cap. + web["asked"].clear() + web["answers"][f"{base}/1"] = b"one at last" + items, state = _run(monkeypatch, module, state) + assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"] + assert [i.body for i in items] == ["one at last", "", "", ""] + assert json.loads(state["ats:detailed:fake:acme"]) == [ + "fake:acme:1", "fake:acme:3", "fake:acme:4"] + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] + + +def test_a_429_on_a_detail_does_not_mark_the_role_as_failing( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)}) + _items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert json.loads(state["ats:detailfailed:fake:acme"]) == [] + assert json.loads(state["ats:detailed:fake:acme"]) == [] + + +def test_failed_ids_no_longer_listed_leave_the_failed_list( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")}) + state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} + _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] + + +def test_detail_fills_only_the_fields_a_later_pull_keeps( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], + caplog: pytest.LogCaptureFixture) -> None: + """`salary_text`, `department` and `title` would be rewritten from the + listing on the next pull, so detail does not fill them.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1"]'}) + + def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: + return {"description": "in full", "remote": "hybrid", "currency": "EUR", + "salary_min": 60000, "salary_max": 70000, + "posted": datetime(2026, 9, 1, tzinfo=UTC), + "salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales", + "title": "Another title"} + + caplog.set_level("DEBUG", logger=ats.log.name) + [item], _state = _run(monkeypatch, _fake_host(detail=detail)) + assert (item.body, item.title, item.published_at) == ( + "in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC)) + assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"], + item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000) + assert (item.raw["salary_text"], item.raw["department"]) == ("", "") + assert "department, salary_text, title left out" in caplog.text + assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency", + "salary_min", "salary_max", "posted"} + + +def _paged_host() -> types.ModuleType: + """A host `fake` that pages its listing, one id a page, and has `detail`.""" + module: Any = _fake_host(detail=_detail) + del module.feed_url, module.parse + + def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]: + index = int(cursor or 0) + ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}")) + return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None + + module.page = page + module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug) + assert not atshosts.problems(module, "fake") + return module + + +def test_a_paged_host_reads_every_page_then_each_role_once( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]', + f"{base}/1": b"one", f"{base}/2": b"two"}) + items, state = _run(monkeypatch, _paged_host()) + assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1", + f"{base}/1", f"{base}/2"] + assert [i.body for i in items] == ["one", "two"] + assert state["ats:complete:fake:acme"] == "1" + + web["asked"].clear() + items, state = _run(monkeypatch, _paged_host(), state) + assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"] + assert [i.body for i in items] == ["", ""] + + +def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429), + f"{base}/1": b"one"}) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]), + "ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])} + _items, state = _run(monkeypatch, _paged_host(), state) + assert state["ats:complete:fake:acme"] == "" + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"] + + +def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + """The consumer stops after the first role, so the second, read but not + stored, is read again on the next pull.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"}) + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) + source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) + roles: Any = source.fetch() # a generator, which `close` stops + assert next(roles).body == "one" + roles.close() + assert "ats:detailed:fake:acme" not in source.state + + +# ------------------------------------------------------------ robots.txt + +ROBOTS = "https://jobs.fake.example/robots.txt" + + +def test_robots_that_allow_the_page_let_the_board_be_read( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n", + "https://jobs.fake.example/acme/list": b'["1"]'}) + items, state = _run(monkeypatch, _fake_host(robots=True)) + assert len(items) == 1 + assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True + + +@pytest.mark.parametrize("robots", [ + b"User-agent: *\nDisallow: /acme/\n", + # A rule for this app by name counts as well as the one for everyone. + f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n" + .encode(), +]) +def test_robots_that_disallow_the_page_fail_the_board( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None: + web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'}) + items, state = _run(monkeypatch, _fake_host(robots=True)) + assert items == [] + assert web["asked"] == [ROBOTS] # the listing is never fetched + assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it" + assert "ats:ok:fake:acme" not in state + + +@pytest.mark.parametrize(("robots", "page", "allowed"), [ + # The longest matching rule wins, whatever the order of the lines. + ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", + "https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True), + ("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True), + ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False), + # An allow wins a tie. + ("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True), + # `*` inside a path, and the query counts as part of it. + ("User-agent: *\nDisallow: /*company\n", + "https://career5.successfactors.eu/career?company=x", False), + # `$` anchors the end. + ("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False), + ("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True), + # An empty Disallow allows. + ("User-agent: *\nDisallow:\n", "https://x.example/anything", True), + # A byte order mark and CRLF line ends. + ("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False), + # Paths are case-sensitive. + ("User-agent: *\nDisallow: /Novonesis_Careers/\n", + "https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False), + ("User-agent: *\nDisallow: /Novonesis_Careers/\n", + "https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True), + # A group for this app by name counts, a group for someone else does + # not, and a token matches whole, not as a substring. + (f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n", + "https://x.example/jobs", False), + ("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True), + ("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True), + # Two agents share one group, and a comment is no rule. + ("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n", + "https://x.example/jobs/1", False), + # A group for this app that allows does not undo the `*` group's refusal. + (f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n", + "https://x.example/jobs", False), + # Rules before any user-agent line belong to no group. + ("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True), +]) +def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None: + assert ats._robots_allows(robots.encode(), page) is allowed + + +def test_a_robots_verdict_is_kept_for_a_day( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", + "https://jobs.fake.example/acme/list": b'["1"]'}) + module = _fake_host(robots=True) + _items, state = _run(monkeypatch, module) + assert web["asked"] == [ROBOTS] + + # Within the day the kept verdict holds, even though the site changed. + web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n" + web["asked"].clear() + items, state = _run(monkeypatch, module, state) + assert (items, web["asked"]) == ([], []) + + # A day on, robots.txt is read again. + kept = json.loads(state["ats:robots:fake:acme"]) + kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat() + state["ats:robots:fake:acme"] = json.dumps(kept) + items, state = _run(monkeypatch, module, state) + assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"] + assert len(items) == 1 and "ats:failing:fake:acme" not in state + + +@pytest.mark.parametrize("code", [404, 410, 401, 403]) +def test_a_robots_txt_answering_4xx_allows_and_is_kept( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None: + web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'}) + items, state = _run(monkeypatch, _fake_host(robots=True)) + assert len(items) == 1 + assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True + + +@pytest.mark.parametrize("failure", [ + _status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"), + httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"), +]) +def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: + web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'}) + items, state = _run(monkeypatch, _fake_host(robots=True)) + assert items == [] and web["asked"] == [ROBOTS] + assert json.loads(state["ats:failing:fake:acme"])["reason"] == \ + "robots.txt could not be read" + assert "ats:robots:fake:acme" not in state + + +def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]: + """A site whose robots.txt on jobs.fake.example redirects to `location`, + where a robots.txt that disallows everything is served.""" + def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == ROBOTS: + return httpx.Response(301, headers={"location": location}) + if request.url.path == "/robots.txt": + return httpx.Response(200, text="User-agent: *\nDisallow: /\n") + return httpx.Response(200, text='["1"]') + return handler + + +def test_a_robots_txt_redirect_to_the_www_twin_is_followed( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ + pytest.raises(ats._Disallowed): + ats._check_robots(client, "fake:acme", {}) + + +@pytest.mark.parametrize("location", [ + "https://evil.example/robots.txt", "https://other.fake.example/robots.txt", + "http://www.jobs.fake.example/robots.txt", +]) +def test_a_robots_txt_redirect_anywhere_else_is_unreachable( + monkeypatch: pytest.MonkeyPatch, location: str) -> None: + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable): + ats._check_robots(client, "fake:acme", {}) + + +def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ + pytest.raises(ats._Refused): + ats._get(client, ROBOTS, "fake", "acme") + + +def test_fetch_board_honours_robots_txt( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", + "https://jobs.fake.example/acme/list": b'["1"]'}) + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + with pytest.raises(ats._Disallowed) as caught: + ats.fetch_board("fake:acme") + assert ats.failure_reason(caught.value) == "robots.txt disallows it" + + +# ----------------------------------------------------------------- Lever EU + +def test_a_lever_slug_may_name_the_eu_instance() -> None: + assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom") + assert not ats.is_known_target("lever:us/acme") + assert not ats.is_known_target("lever:eu/") + assert not ats.is_known_target("lever:eu/a/b") + assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme" + assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom" + assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json" + eu = ats._allowed_host("lever", "eu/tomtom") + assert eu("api.eu.lever.co") and not eu("api.lever.co") + assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co") + + +def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None: + urls: list[str] = [] + + def fake_get(client, url, host, slug, *, method="GET"): + urls.append(url) + assert ats._allowed_host(host, slug)(httpx.URL(url).host) + return (FIXTURES / "lever.json").read_bytes() + + monkeypatch.setattr(ats, "_get", fake_get) + rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type] + assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"] + assert rows and complete + + +def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"}) + + with _client(handler) as client, pytest.raises(ats._Refused): + ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x") diff --git a/tests/test_discover.py b/tests/test_discover.py index 5c1b581..a022662 100644 --- a/tests/test_discover.py +++ b/tests/test_discover.py @@ -4,7 +4,11 @@ site's pages are read.""" from __future__ import annotations import json +import re +import types +from collections.abc import Set from pathlib import Path +from typing import Any import httpx import pytest @@ -36,16 +40,59 @@ def test_boards_in_names_readable_hosts_with_their_slug() -> None: def test_boards_in_keeps_hosts_the_app_cannot_read() -> None: page = """ - Workday - SF + Taleo TT """ assert boards_in(page) == [ - ("workday", "novo", "https://novo.wd3.myworkdayjobs.com"), - ("successfactors", "", "performancemanager.successfactors.eu"), + ("taleo", "", ".taleo.net"), ("teamtailor", "acme", "https://acme.teamtailor.com")] +def test_boards_in_names_a_lever_eu_board_by_its_region() -> None: + page = ('EU' + '' + 'US') + assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"), + ("lever", "whereby")] + + +def test_boards_in_lowercases_a_slug_unless_the_host_keeps_its_case( + monkeypatch: pytest.MonkeyPatch) -> None: + """A Workday-shaped host with two groups and `KEEP_CASE`: the site keeps + its case, and a site called `jobs` is a site, since only group 1 is + checked against the hosts' own path words.""" + workday = re.compile(r"https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/" + r"(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.IGNORECASE) + monkeypatch.setattr(mod, "HOSTS", (("workday", workday), + *(h for h in mod.HOSTS if h[0] != "workday"))) + monkeypatch.setattr(mod, "_KEEP_CASE", frozenset({"workday"})) + page = ('a' + 'b' + 'c' + 'd') + assert [(h, s) for h, s, _ in boards_in(page)] == [ + ("workday", "lego.wd103/LEGO_External"), ("workday", "redhat.wd5/jobs"), + ("smartrecruiters", "boschgroup"), ("lever", "eu/tomtom")] + # A host's own path word in group 1 still names no board. + assert boards_in('x') == [] + + +def test_no_module_of_the_eight_keeps_its_case() -> None: + """SmartRecruiters and Ashby ignore case (checked live, 2026-09-29), so a + slug from a page stays lowercased as it always was.""" + assert not mod._KEEP_CASE + + +def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None: + """Wave by wave, a host named here becomes readable by its module alone.""" + taleo: Any = types.ModuleType("taleo") + taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",) + hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE) + assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [ + r"https?://([a-z0-9-]+)\.taleo\.net"] + assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE + + def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None: page = ('a' 'b') @@ -81,7 +128,7 @@ class Site: is a page that did not answer. Remembers the order of fetches.""" def __init__(self, pages: dict[str, tuple[str, str] | str], - boards: set[tuple[str, str]] = frozenset()) -> None: + boards: Set[tuple[str, str]] = frozenset()) -> None: self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()} self.boards = boards self.fetched: list[str] = [] @@ -138,12 +185,12 @@ def test_a_careers_pages_own_link_is_followed_one_level_down() -> None: def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None: site = Site({ "https://acme.com/": "

welcome

", - "https://acme.com/careers": 'Jobs', + "https://acme.com/careers": 'Jobs', }, boards={("greenhouse", "acme")}) found = run(site) - assert found == Discovery(ats="workday", ats_slug="acme", + assert found == Discovery(ats="taleo", ats_slug="", careers_url="https://acme.com/careers", - evidence="https://acme.wd3.myworkdayjobs.com") + evidence=".taleo.net") assert not found.readable # The page named the host: a probe could only find another tenant's # board of the same name, like the greenhouse:acme this fake web has. @@ -223,6 +270,59 @@ def test_slug_probes_find_a_board_the_pages_never_link() -> None: assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme" +@pytest.mark.parametrize(("boards", "found"), [ + ({("greenhouse", "acme"), ("ashby", "acme")}, "greenhouse:acme"), + ({("lever", "acme"), ("ashby", "acme")}, "lever:acme"), + ({("smartrecruiters", "acme"), ("personio", "acme")}, "smartrecruiters:acme"), + ({("lever", "eu/acme"), ("personio", "acme")}, "lever:eu/acme"), +]) +def test_a_slug_on_two_hosts_goes_to_the_one_probed_first( + boards: set[tuple[str, str]], found: str) -> None: + site = Site({}, boards=boards) + assert run(site).board == found + + +def test_the_probe_order_is_the_one_before_modules() -> None: + assert list(mod._PROBES)[:7] == ["greenhouse", "lever", "ashby", "smartrecruiters", + "recruitee", "personio", "teamtailor"] + site = Site({}) + run(site) + assert site.probed[:8] == [("greenhouse", "acme"), ("lever", "acme"), ("lever", "eu/acme"), + ("ashby", "acme"), ("smartrecruiters", "acme"), + ("recruitee", "acme"), ("personio", "acme"), + ("teamtailor", "acme")] + + +def test_a_lever_eu_board_is_found_by_its_probe() -> None: + site = Site({"https://acme.com/": "

a script draws the jobs

"}, + boards={("lever", "eu/acme")}) + found = run(site) + assert found.board == "lever:eu/acme" + assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json" + assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme")) + + +def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None: + asked: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + asked.append(str(request.url)) + if request.url.host == "api.eu.lever.co": + return httpx.Response(200, json=[]) + if request.url.host == "api.ashbyhq.com": + return httpx.Response(200, content=b'{"jobs":[]}') + return httpx.Response(404) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + assert mod.probe(client, "lever", "eu/acme") + assert not mod.probe(client, "lever", "acme") + assert not mod.probe(client, "ashby", "acme") # an empty listing is no board + assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json", + "https://api.lever.co/v0/postings/acme?limit=1&mode=json", + "https://api.ashbyhq.com/posting-api/job-board/acme"] + assert "workable" not in mod._PROBES and "lever" in mod._PROBES + + def test_a_site_that_never_answers_gives_an_empty_discovery() -> None: site = Site({}) assert run(site) == Discovery() @@ -251,7 +351,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa {"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen", "board": "ashby:pleo"}, {"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK", - "city": "Bagsværd", "ats": "successfactors", + "city": "Bagsværd", "ats": "taleo", "careers_url": "https://www.novonordisk.com/careers.html"}, {"name": "Odd", "board": "nosuch:odd"}, ]})) @@ -266,7 +366,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa assert (sub.target, sub.company) == ("ashby:pleo", "Pleo") novo = store.company("novo-nordisk") assert novo is not None - assert (novo.ats, novo.careers_url) == ("successfactors", + assert (novo.ats, novo.careers_url) == ("taleo", "https://www.novonordisk.com/careers.html") odd = store.company("odd") assert odd is not None and odd.ats == "" diff --git a/tests/test_sources.py b/tests/test_sources.py index dccc4b5..754f9bc 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -546,6 +546,24 @@ def test_replacing_or_detaching_a_board_forgets_that_it_was_failing( assert failing_boards(store) == {} +def test_replacing_or_detaching_a_board_forgets_its_robots_and_detail_records( + store: Store, boards: Boards) -> None: + kept = ("ats:robots:", "ats:detailed:", "ats:detailfailed:") + sources.set_board(store, "acme", "greenhouse:acme", name="Acme") + for key in kept: + store.state[key + "greenhouse:acme"] = "[]" + store.state["ats:robots:lever:other"] = "{}" + + sources.set_board(store, "acme", "ashby:acme") + assert not any(key + "greenhouse:acme" in store.state for key in kept) + assert "ats:robots:lever:other" in store.state + + for key in kept: + store.state[key + "ashby:acme"] = "[]" + sources.detach_board(store, "acme") + assert not any(key + "ashby:acme" in store.state for key in kept) + + def test_search_params_keeps_only_what_the_provider_reads() -> None: assert sources.search_params("jobindex", " python ", area=1, region="CaRE_1nn_cSU") == { "q": "python", "area": 1}