Merge branch 'wp0-hosts'
This commit is contained in:
commit
d4fc1ac487
20 files changed
+2760
-913
No files matched your search
+97
-88
@@ -5,10 +5,12 @@ The seed list and the LinkedIn flow both start from a company name and a
|
||||
website, not from a board. This module gets from the one to the other: it
|
||||
fetches the site's careers page, or the pages the home page links to as
|
||||
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
|
||||
host (one the `ats` plugin parses) becomes a `board` target the pull can
|
||||
read; any other host is recorded as `ats`, with its slug where the URL
|
||||
carries one, so the company is kept as one the app cannot read yet rather
|
||||
than dropped.
|
||||
host (one with a module in `plugins/atshosts/`, which the `ats` plugin
|
||||
parses) becomes a `board` target the pull can read; any other host is
|
||||
recorded as `ats`, with its slug where the URL carries one, so the company is
|
||||
kept as one the app cannot read yet rather than dropped. The patterns and
|
||||
probes of a readable host are its module's; the hosts only named are in
|
||||
`_UNREADABLE` here, and one of them that gains a module is read from then on.
|
||||
|
||||
A Teamtailor site on the company's own domain names no slug: the page only
|
||||
loads Teamtailor's CDN. The board there is the site's own host, and it is
|
||||
@@ -16,7 +18,8 @@ taken only once that host answers `/jobs.rss` with a Teamtailor feed.
|
||||
|
||||
Only when no page names a host at all are the hosts with a cheap per-slug
|
||||
endpoint (`_PROBES`) probed with the company's own slug, because a site that
|
||||
draws its jobs from an iframe or a script leaves no link to find. A page that
|
||||
draws its jobs from an iframe or a script leaves no link to find. Lever is
|
||||
probed on its EU instance as well, as `lever:eu/<slug>`. A page that
|
||||
names an unreadable host is the answer: a probe on a common name can hit
|
||||
another tenant's board.
|
||||
|
||||
@@ -37,13 +40,15 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from types import ModuleType
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import USER_AGENT
|
||||
from plugins import atshosts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -53,55 +58,61 @@ MAX_BYTES = 2 << 20
|
||||
MAX_PAGES = 6
|
||||
|
||||
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
|
||||
READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||
"recruitee", "personio", "teamtailor")
|
||||
READABLE: tuple[str, ...] = tuple(atshosts.HOSTS)
|
||||
|
||||
#: host -> pattern over a URL. Group 1, where there is one, is the slug the
|
||||
#: host names the company by; a host with no group is recognised but its
|
||||
#: board has no slug the app could use. Order matters only for readability:
|
||||
#: the readable hosts come first so the first match on a page is the one the
|
||||
#: app can read.
|
||||
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
|
||||
(host, re.compile(pattern, re.IGNORECASE)) for host, pattern in (
|
||||
("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"),
|
||||
("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"),
|
||||
("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"),
|
||||
("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"),
|
||||
("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"),
|
||||
("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"),
|
||||
("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"),
|
||||
("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"),
|
||||
("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"),
|
||||
("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"),
|
||||
("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"),
|
||||
("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"),
|
||||
# No slug: a career site on the company's own domain, which only loads
|
||||
# Teamtailor's CDN. `_discover` reads its board off the page's host.
|
||||
("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"),
|
||||
# Not readable. The slug, where the URL has one, is kept for the day
|
||||
# a parser exists.
|
||||
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
|
||||
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
|
||||
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
|
||||
("taleo", r"\.taleo\.net"),
|
||||
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
|
||||
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
|
||||
("join", r"join\.com/companies/([a-z0-9-]+)"),
|
||||
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
|
||||
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
|
||||
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
|
||||
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
|
||||
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
|
||||
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
|
||||
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
|
||||
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
|
||||
("eightfold", r"\.eightfold\.ai"),
|
||||
("phenom", r"phenompeople\.com|phenom\.com/careers"),
|
||||
("avature", r"\.avature\.net"),
|
||||
("oracle", r"oraclecloud\.com/hcmUI"),
|
||||
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
|
||||
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
|
||||
))
|
||||
#: host -> pattern over a URL, for the hosts discovery names but the app
|
||||
#: cannot read. Group 1, where there is one, is the slug the host names the
|
||||
#: company by, kept for the day a parser exists; a host with no group is
|
||||
#: recognised but its board has no slug the app could use.
|
||||
_UNREADABLE: tuple[tuple[str, str], ...] = (
|
||||
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
|
||||
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
|
||||
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
|
||||
("taleo", r"\.taleo\.net"),
|
||||
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
|
||||
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
|
||||
("join", r"join\.com/companies/([a-z0-9-]+)"),
|
||||
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
|
||||
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
|
||||
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
|
||||
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
|
||||
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
|
||||
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
|
||||
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
|
||||
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
|
||||
("eightfold", r"\.eightfold\.ai"),
|
||||
("phenom", r"phenompeople\.com|phenom\.com/careers"),
|
||||
("avature", r"\.avature\.net"),
|
||||
("oracle", r"oraclecloud\.com/hcmUI"),
|
||||
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
|
||||
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
|
||||
)
|
||||
|
||||
|
||||
def _hosts(modules: Mapping[str, ModuleType],
|
||||
unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]:
|
||||
"""Every host's patterns, compiled: the readable hosts' from their
|
||||
modules first, so the first match at one place on a page is the one the
|
||||
app can read, then the unreadable ones. An unreadable host that has
|
||||
gained a module is left out here, since its module's patterns replace
|
||||
it."""
|
||||
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
|
||||
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
|
||||
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
|
||||
|
||||
|
||||
#: host -> pattern over a URL, every host discovery can name. The slug is
|
||||
#: group 1, or, for a pattern with more groups, the groups that matched
|
||||
#: joined by "/" (Lever's `eu/<site>`).
|
||||
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE)
|
||||
|
||||
#: The hosts whose slugs keep the case the URL wrote them in, by their
|
||||
#: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its
|
||||
#: listing and its robots.txt paths are case-sensitive. Every other slug is
|
||||
#: lowercased, which is how a page that writes `BoschGroup` and one that
|
||||
#: writes `boschgroup` name one board.
|
||||
_KEEP_CASE = frozenset(name for name, module in atshosts.HOSTS.items()
|
||||
if getattr(module, "KEEP_CASE", False))
|
||||
|
||||
#: What a Teamtailor feed carries and a page that only looks like one does not:
|
||||
#: the namespace of `tt:locations`, declared on `<rss>` even when no role is open.
|
||||
@@ -125,7 +136,7 @@ _ANCHOR = re.compile(r"<a\b[^>]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)
|
||||
re.IGNORECASE | re.DOTALL)
|
||||
_TAGS = re.compile(r"<[^>]+>")
|
||||
|
||||
#: Words a pattern's slug group can catch that name no board: the hosts'
|
||||
#: Words a pattern's group 1 can catch that name no board: the hosts'
|
||||
#: own path segments, the analytics subdomain Recruitee sites embed, and
|
||||
#: the CDN and the admin app Teamtailor sites load from (cdn.teamtailor.com,
|
||||
#: app.teamtailor.com).
|
||||
@@ -161,9 +172,13 @@ def boards_in(text: str) -> list[Found]:
|
||||
hits: list[tuple[int, str, str, str]] = []
|
||||
for host, pattern in HOSTS:
|
||||
for m in pattern.finditer(text):
|
||||
slug = (m.group(1) if m.groups() and m.group(1) else "").lower()
|
||||
if slug in _NOT_SLUGS:
|
||||
# Only group 1 names a board or a host's own path word; a later
|
||||
# group may be anything, a Workday site called `jobs` included.
|
||||
if m.re.groups and (m.group(1) or "").lower() in _NOT_SLUGS:
|
||||
continue
|
||||
slug = "/".join(part for part in m.groups() if part)
|
||||
if host not in _KEEP_CASE:
|
||||
slug = slug.lower()
|
||||
hits.append((m.start(), host, slug, m.group(0)))
|
||||
for _pos, host, slug, url in sorted(hits):
|
||||
if (host, slug) in seen:
|
||||
@@ -237,25 +252,28 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
|
||||
return "", ""
|
||||
|
||||
|
||||
#: host -> a cheap URL that answers 2xx only when the slug is a board there.
|
||||
#: Workable is left out: its API sits behind a rate limit that a run of
|
||||
#: guesses trips, and a Workable site links its board plainly anyway.
|
||||
_PROBES = {
|
||||
"greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}",
|
||||
"lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json",
|
||||
"ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}",
|
||||
"smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1",
|
||||
"recruitee": "https://{slug}.recruitee.com/api/offers/",
|
||||
"personio": "https://{slug}.jobs.personio.de/xml",
|
||||
"teamtailor": "https://{slug}.teamtailor.com/jobs.rss",
|
||||
}
|
||||
#: host -> a cheap URL that answers 2xx only when the slug is a board there,
|
||||
#: from each module's `probe_url`. Workable has none: its API sits behind a
|
||||
#: rate limit that a run of guesses trips, and a Workable site links its
|
||||
#: board plainly anyway.
|
||||
_PROBES: dict[str, Callable[[str], str]] = {
|
||||
name: module.probe_url for name, module in atshosts.HOSTS.items()
|
||||
if hasattr(module, "probe_url")}
|
||||
|
||||
|
||||
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
|
||||
"""The slugs a guessed name is probed as on `host`: the name itself, and
|
||||
for Lever the name on its EU instance too (the module's `probe_slugs`)."""
|
||||
module = atshosts.HOSTS[host]
|
||||
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
|
||||
|
||||
|
||||
def probe(client: httpx.Client, host: str, slug: str) -> bool:
|
||||
"""Whether `host` has a board called `slug`, by its cheapest endpoint.
|
||||
Ashby and SmartRecruiters answer an empty listing for an unknown slug,
|
||||
so there a listing without jobs counts as no board."""
|
||||
url = _PROBES[host].format(slug=slug)
|
||||
A host that answers an unknown slug with an empty listing (Ashby,
|
||||
SmartRecruiters, ...) says in its module's `probe_ok` what a listing
|
||||
with a board looks like."""
|
||||
url = _PROBES[host](slug)
|
||||
try:
|
||||
response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT})
|
||||
@@ -263,18 +281,8 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool:
|
||||
return False
|
||||
if not response.is_success:
|
||||
return False
|
||||
body = response.content
|
||||
if host == "ashby":
|
||||
return b'"jobs":[' in body and b'"jobs":[]' not in body
|
||||
if host == "smartrecruiters":
|
||||
return b'"totalFound":' in body and b'"totalFound":0' not in body
|
||||
if host == "recruitee":
|
||||
return b'"offers":[' in body and b'"offers":[]' not in body
|
||||
if host == "personio":
|
||||
return b"<position>" in body
|
||||
if host == "teamtailor":
|
||||
return b"<item>" in body
|
||||
return True
|
||||
module = atshosts.HOSTS[host]
|
||||
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
|
||||
|
||||
|
||||
Fetch = Callable[[httpx.Client, str], tuple[str, str]]
|
||||
@@ -377,9 +385,10 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
|
||||
# probe could only add another tenant's board of the same name.
|
||||
if unreadable is not None:
|
||||
return unreadable
|
||||
for slug in slug_guesses(name, site):
|
||||
for guess in slug_guesses(name, site):
|
||||
for host in _PROBES:
|
||||
if probe_slug(client, host, slug):
|
||||
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
|
||||
careers_url=careers_page, evidence=_PROBES[host].format(slug=slug))
|
||||
for slug in _probe_slugs(host, guess):
|
||||
if probe_slug(client, host, slug):
|
||||
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
|
||||
careers_url=careers_page, evidence=_PROBES[host](slug))
|
||||
return Discovery(careers_url=careers_page)
|
||||
+5
-1
@@ -111,8 +111,12 @@ def detach_board(store: Store, key: str) -> int:
|
||||
def _forget_failing(store: Store, target: str) -> None:
|
||||
"""Drop what the pull recorded about a board nobody reads any more: a
|
||||
record of `target` failing would otherwise outlive the board, and count
|
||||
in `app companies check` for good."""
|
||||
in `app companies check` for good. The `ats` plugin's robots.txt verdict
|
||||
and its lists of roles read and failed in detail go with it, since they
|
||||
are about that board alone and would only grow stale."""
|
||||
store.state.pop(FAILING + target, None)
|
||||
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
|
||||
store.state.pop(key + target, None)
|
||||
|
||||
|
||||
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
||||
|
||||
+1
-1
@@ -661,7 +661,7 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
|
||||
#: read as when a role was posted, checked against live boards on
|
||||
#: 2026-09-29:
|
||||
#:
|
||||
#: - `greenhouse`: `plugins.ats._parse_greenhouse` already prefers
|
||||
#: - `greenhouse`: `plugins.atshosts.greenhouse.parse` already prefers
|
||||
#: `first_published` over `updated_at`. A live board (adyen) showed a job
|
||||
#: with `first_published` five months before `updated_at`, proving the
|
||||
#: first stays fixed while the second moves on an edit — the resettable
|
||||
|
||||
@@ -46,32 +46,52 @@ The 49 were missed because the slug is not the company's name (`BoschGroup`, `ji
|
||||
|
||||
## Contracts between packages
|
||||
|
||||
Host module, `plugins/atshosts/<host>.py`:
|
||||
Host module, `plugins/atshosts/<host>.py`, as built by WP0. The docstring of `plugins/atshosts/__init__.py` holds the same contract and is the one to keep current. A module imports `desk` and `._shared` only, never `plugins.ats`.
|
||||
|
||||
```python
|
||||
NAME = "workday" # the host of a target `workday:<slug>`
|
||||
LABEL = "Workday" # the Sources page's select
|
||||
PATTERNS = (r"https?://...",) # discover: over a URL, group 1 is the slug
|
||||
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's file name
|
||||
LABEL = "Workday" # the Sources page's select
|
||||
PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||
RANK = 100 # optional int, default 100; lower comes first
|
||||
KEEP_CASE = True # optional, default False; keep a discovered slug's case
|
||||
|
||||
def slug_ok(slug: str) -> bool: ... # strict; a slug that fails is no target
|
||||
def allowed(slug: str) -> Callable[[str], bool]: ... # hostnames a fetch may reach
|
||||
def careers_url(slug: str) -> str: ... # the page a person opens
|
||||
def robots_page(slug: str) -> str: ... # optional; the URL robots.txt must allow
|
||||
def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch
|
||||
def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach
|
||||
def careers_url(slug: str) -> str: ... # the page a person opens
|
||||
|
||||
# one of the two
|
||||
def feed_url(slug: str) -> str: ...
|
||||
def feed_url(slug: str) -> str: ... # the whole board in one answer
|
||||
def parse(body: bytes, slug: str) -> list[Row]: ...
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page: ...
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page: ... # a paged board, one page a call
|
||||
def rows(decoded: Any, slug: str) -> list[Row]: ...
|
||||
|
||||
# optional
|
||||
def detail(get: Get, slug: str, row: Row) -> Row: ...
|
||||
def probe_url(slug: str) -> str: ...
|
||||
def probe_ok(body: bytes) -> bool: ...
|
||||
def fallback_url(slug: str) -> str: ... # feed hosts: a smaller feed, read once when feed_url is over the byte cap
|
||||
def robots_page(slug: str) -> str: ... # the URL robots.txt must allow; on a host allowed(slug) admits
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: ... # fields from the role's own page
|
||||
def probe_url(slug: str) -> str: ... # discovery: answers 2xx only for a board
|
||||
def probe_ok(body: bytes) -> bool: ... # discovery: when 2xx is not enough
|
||||
def probe_slugs(guess: str) -> tuple[str, ...]: ... # the slugs a guessed name is probed as; default (guess,)
|
||||
```
|
||||
|
||||
`Get` is `get(url, *, method="GET", payload=None, headers=None) -> bytes`. `Row` and `Page` are as in `plugins/ats.py` today. `plugins/atshosts/__init__.py` lists the modules with `pkgutil.iter_modules`, sorted by name, so a new host registers by existing.
|
||||
`Get`, `Row` and `Page` are in `plugins/atshosts/_shared.py`, with the helpers several hosts share: `only(host)` for `allowed`, `path_slug_ok` and `label_slug_ok` for `slug_ok`, `parse_dt`, `is_web_url`, `num`, `yearly_amount` and `pay_line`. `plugins/atshosts/__init__.py` lists the modules with `pkgutil.iter_modules`, checks each against the contract at import (an ImportError names the module and what it lacks), and orders `HOSTS` by `(RANK, NAME)`. The eight first hosts hold ranks 10 to 80, which keeps the order they had before the modules: the select opens on Greenhouse, and a guessed slug on two hosts goes to the one probed first.
|
||||
|
||||
Order and names. `RANK` orders the Sources page's select, the plugin's description and discovery's probes. `NAME` equals the file name. A host that `discover.py` names in its own `_UNREADABLE` table is taken from its module as soon as the module exists, so a wave 1 package does not edit `discover.py`.
|
||||
|
||||
Patterns. The slug is group 1, or, when a pattern has more groups, the groups that matched joined by "/" (`https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)` gives `lego.wd103/LEGO_External`). Only group 1 is checked against discovery's words that name no board (`api`, `jobs`, `embed`, ...). The slug is lowercased unless the module sets `KEEP_CASE = True`, which a host with case-sensitive paths needs, since robots.txt paths are case-sensitive too. A pattern without a group names the host with no slug.
|
||||
|
||||
Slugs. `slug_ok` uses `re.fullmatch` (a `$` lets a trailing newline through) and refuses anything that could add a path, query, fragment, port or user to the URLs built from it. A slug of several parts joins them with "/": Lever's EU boards are `lever:eu/<site>`.
|
||||
|
||||
`get(url, *, method="GET", payload=None, headers=None) -> bytes` is `ats._get` bound to the board. A POST sends `payload` as JSON; `headers` are added to the default User-Agent and Accept and replace either where they name it. It fetches https on port 443 only, never an IP address or a local name (`localhost`, `.local`, `.internal`, `.lan`, `.home.arpa`), follows at most three redirects each checked against `allowed(slug)`, cuts the body at `MAX_BYTES` and ends within `DEADLINE`. It returns the body and raises `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError` on a timeout or a connection failure, and another exception, `ValueError` among them, for a refused host, an oversized body or a broken redirect. A module lets these through.
|
||||
|
||||
`Page` is `(decoded, results, total, cursor)`: the decoded page handed to `rows`, its raw results as dicts with an `id` (counted by distinct id for completeness), the total the host states or None, and the next cursor or None after the last page. The first call gets cursor None. A board with a total is complete when as many distinct ids were read, one without when the read ended on an empty page; a 429 after the first page ends the read, incomplete. A paged module may also have `parse(body, slug)` for one page's body, for its tests.
|
||||
|
||||
A `Row` has `source_id`, `title`, `location`, `url`, `posted` (aware datetime or None), `department`, `description` (plain text), `remote` (True, False, a word or None) and `salary_text`, and, where the host states pay, `salary_min`, `salary_max` (yearly integers) and `currency`.
|
||||
|
||||
`detail` returns a dict of what the role's own page states. The shared code calls it for roles not yet read (`ats:detailed:<target>`), at most `DETAIL_CAP` (200) per board per pull, `PAGE_DELAY` apart, the roles never tried before those whose detail failed (`ats:detailfailed:<target>`). Only `DETAIL_FIELDS` are taken: `description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the listing's empty value (`upsert_role`'s keep-filled columns, the salary figures, and `posted_at`, which keeps the earliest date). Any other key is dropped with a debug log line: `title`, `url`, `department` and `salary_text` cannot be filled from `detail`, since the listing rewrites them every pull. A role not read in detail on a pull is sent with an empty description. A 404 or 410 counts as read, any other failure is retried on a later pull, and a 429 ends the detail reads for that board and pull. Completeness is decided by the listing alone.
|
||||
|
||||
robots.txt. For a module with `robots_page`, the shared code reads `https://<its host>/robots.txt` before the listing, at most once a day per board (`ats:robots:<target>`), following a redirect to the host's `www.` twin or back and no other. It fails the board with "robots.txt disallows it" when the rules for `*` or for the app's own token disallow the page, matched per RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a final `$` in paths, paths case-sensitive, an empty Disallow allows. A 4xx other than 429 means there are no rules, which allows. A 5xx, a 429, a timeout, a connection error or a redirect elsewhere fails the board with "robots.txt could not be read", and the next pull asks again.
|
||||
|
||||
Slugs: Workday `lego.wd103/LEGO_External`; SuccessFactors, Phenom and the sitemap reader take the site's hostname, with a path where the site has one (`jobs.teradyne.com/MiR`); Oracle `<host>/<siteNumber>`.
|
||||
|
||||
|
||||
+7
-5
@@ -1,8 +1,10 @@
|
||||
"""Drop-in plugins, discovered by `plugin_paths` in `config.yaml`.
|
||||
|
||||
A plugin file imports `desk` only — see `tests/test_architecture.py`. This
|
||||
`__init__.py` makes the directory importable as `plugins.<name>` for the rest
|
||||
of roledesk (`jobs/pull.py`, `cli.py`, the tests), which the plugin contract
|
||||
itself never relies on: `desk.sources.registry` finds `ats.py` by path, not
|
||||
by this package.
|
||||
A plugin file imports `desk` only — see `tests/test_architecture.py` — and
|
||||
its own helpers in this directory. This `__init__.py` makes the directory
|
||||
importable as `plugins.<name>`, for the rest of roledesk (`jobs/pull.py`,
|
||||
`cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the
|
||||
package `plugins/atshosts/`. `desk.sources.registry` still finds `ats.py` by
|
||||
path, not by this package; `ats.py` puts the repo root on `sys.path` itself
|
||||
when the app has not, so that `plugins.atshosts` imports either way.
|
||||
"""
|
||||
+440
-789
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,210 @@
|
||||
"""The hosts the `ats` plugin reads, one module each.
|
||||
|
||||
A host registers by existing: every module in this package whose name does
|
||||
not start with `_` is loaded when the package is imported, checked against
|
||||
the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A
|
||||
module that breaks the contract fails the import with an ImportError naming
|
||||
the module and what it lacks, rather than a pull that fails board by board.
|
||||
`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host
|
||||
select and its description from `HOSTS`, and `app/jobs/discover.py` its
|
||||
patterns and probes. The package sits beside `ats.py` rather than being
|
||||
`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and
|
||||
the tests patch names on the `ats` module.
|
||||
|
||||
A module is named after its host (`plugins/atshosts/workday.py`) and imports
|
||||
`desk` and `._shared` only, never `plugins.ats`. It holds:
|
||||
|
||||
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
||||
LABEL = "Workday" # what the Sources page's select shows
|
||||
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||
RANK = 100 # optional int; lower comes first in the select and the probes
|
||||
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
||||
|
||||
def slug_ok(slug: str) -> bool # strict, see below
|
||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
||||
def careers_url(slug: str) -> str # the page a person opens
|
||||
|
||||
and one of two ways to read a board:
|
||||
|
||||
def feed_url(slug: str) -> str # the whole board in one answer,
|
||||
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call,
|
||||
def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page
|
||||
|
||||
and, optionally:
|
||||
|
||||
def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read
|
||||
# once when `feed_url` is over the cap
|
||||
def robots_page(slug: str) -> str # the URL robots.txt must allow
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page
|
||||
def probe_url(slug: str) -> str # discovery: 2xx only for a board
|
||||
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
||||
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
|
||||
|
||||
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
|
||||
hold 10 to 80, which keeps the order they had before there were modules, so
|
||||
a new host comes after them unless it says otherwise. The order decides the
|
||||
select's default and which host wins when discovery's guessed slug exists on
|
||||
two.
|
||||
|
||||
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
|
||||
which lets a trailing newline through) and refuses anything that could add a
|
||||
path, a query, a fragment, a port or a user to the URLs built from it.
|
||||
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
|
||||
(one DNS label) cover the usual cases; a slug with several parts joins them
|
||||
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
|
||||
|
||||
**Patterns.** The slug is group 1, or, when a pattern has more than one
|
||||
group, the groups that matched joined by "/". Only group 1 is checked
|
||||
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
|
||||
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
|
||||
paths are case-sensitive (Workday's site), since robots.txt paths are too. A
|
||||
pattern without a group names the host with no slug. A host that
|
||||
`discover.py` names in its own `_UNREADABLE` table is taken from its module
|
||||
instead as soon as the module exists.
|
||||
|
||||
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
||||
name; `_shared.only(host)` is the usual one. Every request goes through
|
||||
`get`, which is `ats._get` bound to the board:
|
||||
|
||||
get(url, *, method="GET", payload=None, headers=None) -> bytes
|
||||
|
||||
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
|
||||
default User-Agent and `Accept: application/json, text/xml`, and replace
|
||||
either where they name it. `get` is https only on port 443, never to an IP
|
||||
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
|
||||
most three redirects each checked against `allowed(slug)`, cuts the body at
|
||||
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
|
||||
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
|
||||
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
|
||||
failure, and another exception (`ValueError` among them) for a refused host,
|
||||
an oversized body or a broken redirect. A module lets these through; the
|
||||
shared code logs, records and decides.
|
||||
|
||||
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
|
||||
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
|
||||
id, to decide completeness), the total the host states or None, and the
|
||||
cursor of the next page or None after the last. The first call gets cursor
|
||||
None. A board with a total is complete when as many distinct ids were read;
|
||||
one without, when the read ended on an empty page. A 429 after the first page
|
||||
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
|
||||
apart. A paged module may also have `parse(body, slug)` for one page's body.
|
||||
|
||||
**Rows.** A `Row` is a dict with `source_id` (the host's own id for the
|
||||
role, unique on the board), `title`, `location`, `url` (http or https,
|
||||
checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see
|
||||
`_shared.parse_dt`), `department`, `description` (plain text, see
|
||||
`desk.sources.strip_html`), `remote` (True, False, a word or None) and
|
||||
`salary_text`; and, where the host states pay, `salary_min`, `salary_max`
|
||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`.
|
||||
|
||||
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
||||
dict of the fields it found. The shared code calls it for roles not yet read
|
||||
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
|
||||
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
|
||||
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
|
||||
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
|
||||
and `posted`, the keys whose columns survive a later pull that sends the
|
||||
listing's empty value; any other key is left out with a debug log line.
|
||||
`title`, `url`, `department` and `salary_text` cannot be filled from
|
||||
`detail`: the listing rewrites them every pull. A role not read in detail
|
||||
on a pull is sent with an empty description, so the one read earlier stays.
|
||||
A 404 or a 410 counts as read; any other failure is logged and retried on a
|
||||
later pull; a 429 ends the detail reads of that board for the pull.
|
||||
Completeness is decided by the listing alone.
|
||||
|
||||
**robots.txt.** With `robots_page(slug)`, the shared code reads
|
||||
`https://<its host>/robots.txt` through `get` before the listing, at most
|
||||
once a day per board (`ats:robots:<target>`), following a redirect to the
|
||||
host's `www.` twin or back and no other. The board fails with "robots.txt
|
||||
disallows it" when the rules for `*` or for the app's own token disallow the
|
||||
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
|
||||
final `$` in paths, case-sensitive paths). A 4xx other than 429 means no
|
||||
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
|
||||
the board with "robots.txt could not be read" and is asked again next pull.
|
||||
`robots_page` must be on a host `allowed(slug)` admits.
|
||||
|
||||
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
||||
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
||||
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
|
||||
name is probed as, `(guess,)` without it. A host without `probe_url` is never
|
||||
probed, only found by a link.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from types import ModuleType
|
||||
|
||||
#: The rank of a module that states none: after the eight first hosts, whose
|
||||
#: ranks (10 to 80) keep the order they had before there were modules.
|
||||
DEFAULT_RANK = 100
|
||||
|
||||
#: What every host module defines.
|
||||
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
||||
|
||||
#: The functions a module may define; each must be callable if present.
|
||||
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
||||
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
|
||||
|
||||
|
||||
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
||||
"""What `module` lacks or gets wrong against the contract, one phrase
|
||||
each; empty for a module that keeps it. `name` is the module's own name
|
||||
in the package, which `NAME` must equal."""
|
||||
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
|
||||
if name is not None and getattr(module, "NAME", name) != name:
|
||||
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
||||
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
|
||||
out.append("LABEL is not a non-empty string")
|
||||
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
||||
out.append("KEEP_CASE is not a bool")
|
||||
rank = getattr(module, "RANK", DEFAULT_RANK)
|
||||
if not isinstance(rank, int) or isinstance(rank, bool):
|
||||
out.append("RANK is not an int")
|
||||
patterns = getattr(module, "PATTERNS", ())
|
||||
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
|
||||
out.append("PATTERNS is not a tuple of strings")
|
||||
else:
|
||||
for pattern in patterns:
|
||||
try:
|
||||
re.compile(pattern)
|
||||
except (TypeError, re.error) as exc:
|
||||
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
|
||||
out += [f"{attr} is not callable" for attr in FUNCTIONS
|
||||
if hasattr(module, attr) and not callable(getattr(module, attr))]
|
||||
if hasattr(module, "page"):
|
||||
if not hasattr(module, "rows"):
|
||||
out.append("page without rows")
|
||||
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
|
||||
out.append("neither feed_url and parse nor page and rows")
|
||||
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
|
||||
out.append("probe_ok without probe_url")
|
||||
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
|
||||
out.append("probe_slugs without probe_url")
|
||||
return out
|
||||
|
||||
|
||||
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
|
||||
"""`NAME -> module` for every module of `package` (found on `path`) whose
|
||||
name does not start with `_`, in order of `(RANK, NAME)`. Raises
|
||||
ImportError naming the first module that breaks the contract."""
|
||||
modules: list[ModuleType] = []
|
||||
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
|
||||
if info.name.startswith("_"):
|
||||
continue
|
||||
module = importlib.import_module(f"{package}.{info.name}")
|
||||
if found := problems(module, info.name):
|
||||
raise ImportError(f"{package}.{info.name} breaks the host contract: "
|
||||
+ "; ".join(found))
|
||||
modules.append(module)
|
||||
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
|
||||
return {module.NAME: module for module in modules}
|
||||
|
||||
|
||||
#: host -> its module, in order of `(RANK, NAME)`.
|
||||
HOSTS: dict[str, ModuleType] = load(__path__, __name__)
|
||||
@@ -0,0 +1,192 @@
|
||||
"""What the host modules share: the row and page shapes, the fetch they are
|
||||
handed, and the date, URL and salary helpers more than one host needs.
|
||||
|
||||
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
|
||||
import it without either importing the other. The leading underscore keeps
|
||||
`plugins.atshosts` from listing it as a host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
#: What every parser turns a feed into: one dict per row, with the keys
|
||||
#: `source_id`, `title`, `location`, `url`, `posted`, `department`,
|
||||
#: `description`, `remote` and `salary_text`, and, where a host states pay,
|
||||
#: `salary_min`, `salary_max` and `currency`.
|
||||
Row = dict[str, Any]
|
||||
|
||||
#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded
|
||||
#: page, its raw results (dicts with an `id`, counted for completeness), the
|
||||
#: total it states (None for a host that states none), and the cursor of the
|
||||
#: next page (None after the last). The first call gets cursor None.
|
||||
Page = tuple[Any, list[dict[str, Any]], Any, Any]
|
||||
|
||||
|
||||
class Get(Protocol):
|
||||
"""The fetch a host is handed: `ats.py`'s own, bound to one board, so the
|
||||
allow-list, the redirect rule, the byte cap and the deadline apply to
|
||||
every request a host makes. A POST sends `payload` as JSON; `headers` are
|
||||
added to (and may replace) the default User-Agent and Accept. Returns the
|
||||
body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see
|
||||
"Fetching" in the package's docstring for the rest."""
|
||||
|
||||
def __call__(self, url: str, *, method: str = "GET",
|
||||
payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes: ...
|
||||
|
||||
|
||||
def is_web_url(url: str) -> bool:
|
||||
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
||||
`javascript:` or `data:` link must never reach an href. Port of
|
||||
openings.go's `webURL`."""
|
||||
url = (url or "").strip()
|
||||
if not url:
|
||||
return False
|
||||
parsed = httpx.URL(url)
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
||||
|
||||
|
||||
def only(host: str) -> Callable[[str], bool]:
|
||||
"""The allow-list of a board that is fetched from one host name only,
|
||||
which is every host so far. A host name has no case, and a slug that
|
||||
names one (`Acme` of acme.recruitee.com) may be written with capitals."""
|
||||
want = host.lower()
|
||||
return lambda h: h.lower() == want
|
||||
|
||||
|
||||
#: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`).
|
||||
#: A slug that is one URL path segment: what the hosts that put the slug in
|
||||
#: the path accept, and nothing that could add a path, a query or a fragment.
|
||||
_SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+")
|
||||
|
||||
#: A slug that is one DNS label, for the hosts that put it in the host name.
|
||||
_LABEL = re.compile(r"[A-Za-z0-9-]+")
|
||||
|
||||
|
||||
def path_slug_ok(slug: str) -> bool:
|
||||
return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..")
|
||||
|
||||
|
||||
def label_slug_ok(slug: str) -> bool:
|
||||
return bool(_LABEL.fullmatch(slug))
|
||||
|
||||
|
||||
def parse_dt(value: Any) -> datetime | None:
|
||||
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
||||
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
|
||||
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, int | float):
|
||||
if value <= 0:
|
||||
return None
|
||||
seconds = value / 1000 if value > 10**12 else value
|
||||
return datetime.fromtimestamp(seconds, tz=UTC)
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
||||
if not match:
|
||||
try:
|
||||
parsed = parsedate_to_datetime(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
# ---------------------------------------------------------- structured pay
|
||||
#
|
||||
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
|
||||
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
|
||||
# `raw` into, per docs/decisions.md's "raw is transport, columns are
|
||||
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
|
||||
# structured pay on their list endpoint; Workable, SmartRecruiters and
|
||||
# Teamtailor do not (checked live, 2026-09-29), so their rows leave the
|
||||
# columns out and `ATSSource.fetch` reads a plain '' or None for them.
|
||||
|
||||
|
||||
def num(value: Any) -> float | None:
|
||||
"""A positive number from whatever a board sends (int, float, or a
|
||||
numeral string), or None for anything else, including zero and a
|
||||
negative — a board's placeholder 0 (an unfilled field, not a real
|
||||
figure) must not become a salary floor of zero."""
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if number > 0 else None
|
||||
|
||||
|
||||
def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
|
||||
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
|
||||
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
|
||||
None, "")` when there is no currency, no bound, or `interval` is not one
|
||||
of `factors`' own keys — which is how an hourly or a daily rate is left
|
||||
out rather than annualised into a number nobody stated. `factors` maps
|
||||
the board's own interval spelling to the multiplier that makes it
|
||||
yearly: 1 for a figure already yearly, 12 for a monthly one."""
|
||||
code = str(currency or "").strip().upper()
|
||||
factor = factors.get(str(interval or "").strip())
|
||||
if not code or not factor:
|
||||
return None, None, ""
|
||||
lo, hi = num(min_value), num(max_value)
|
||||
if lo is None and hi is None:
|
||||
return None, None, ""
|
||||
return (int(lo * factor) if lo is not None else None,
|
||||
int(hi * factor) if hi is not None else None, code)
|
||||
|
||||
|
||||
# ------------------------------------------------------------- salary_text
|
||||
#
|
||||
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
|
||||
# hourly and daily rates, a Greenhouse role with several pay zones, and a
|
||||
# range with no currency are all dropped from those columns rather than
|
||||
# guessed into a wrong yearly figure. `salary_text` is where that dropped
|
||||
# information survives — one human-readable line per parser, written
|
||||
# whether or not the pay conformed, so a person can read it and do their own
|
||||
# sums and a stricter analysis can find the roles the columns left out
|
||||
# (`Role.salary_state`, `app/models.py`).
|
||||
|
||||
#: Formats a number the way a salary line writes it: no decimals for a whole
|
||||
#: figure (the normal case for a yearly or monthly one), two for anything
|
||||
#: else (an hourly or daily rate quoted with cents).
|
||||
def fmt_amount(value: float) -> str:
|
||||
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
|
||||
|
||||
|
||||
def bounds_words(lo: float | None, hi: float | None) -> str:
|
||||
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
|
||||
bound stated, or "" with neither."""
|
||||
if lo is not None and hi is not None:
|
||||
return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}"
|
||||
if lo is not None:
|
||||
return f"from {fmt_amount(lo)}"
|
||||
if hi is not None:
|
||||
return f"up to {fmt_amount(hi)}"
|
||||
return ""
|
||||
|
||||
|
||||
def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
|
||||
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
|
||||
per year" — or "" when there is no bound to report at all. A currency
|
||||
that was not stated does not drop the line, unlike the normalised
|
||||
columns: it is said plainly instead, "80,000 to 100,000 per year (no
|
||||
currency stated)", so the figure is not lost along with the currency."""
|
||||
bounds = bounds_words(lo, hi)
|
||||
if not bounds:
|
||||
return ""
|
||||
tail = f" per {period}" if period else ""
|
||||
code = str(currency or "").strip().upper()
|
||||
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Ashby: `ashby:<job board name>`, the whole board in one JSON answer. A
|
||||
board name may carry a dot (`jimdo.com`) or be a uuid.
|
||||
|
||||
The field map is ported from `openings.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
|
||||
|
||||
NAME = "ashby"
|
||||
LABEL = "Ashby"
|
||||
RANK = 30
|
||||
PATTERNS = (
|
||||
r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)",
|
||||
r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)",
|
||||
)
|
||||
|
||||
_API = "api.ashbyhq.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return path_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://jobs.ashbyhq.com/{slug}"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{_API}/posting-api/job-board/{slug}?includeCompensation=true"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return f"https://{_API}/posting-api/job-board/{slug}"
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
"""Ashby answers an empty listing for an unknown name, so a listing
|
||||
without jobs counts as no board."""
|
||||
return b'"jobs":[' in body and b'"jobs":[]' not in body
|
||||
|
||||
|
||||
def _salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
||||
"""The yearly range from `compensation.summaryComponents` (needs
|
||||
`?includeCompensation=true` on the feed URL, see `feed_url`): the first
|
||||
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
|
||||
an hourly one is left out rather than annualised."""
|
||||
for component in components:
|
||||
if component.get("compensationType") != "Salary":
|
||||
continue
|
||||
return yearly_amount(component.get("minValue"), component.get("maxValue"),
|
||||
component.get("currencyCode"), component.get("interval"),
|
||||
{"1 YEAR": 1, "1 MONTH": 12})
|
||||
return None, None, ""
|
||||
|
||||
|
||||
def _salary_text(compensation: dict[str, Any]) -> str:
|
||||
"""One line per `summaryComponents` entry that states a bound, labelled
|
||||
by the board's own `compensationType` when more than one does (a salary
|
||||
range beside a bonus range, say); with just the one, the label is left
|
||||
off, the same as Greenhouse's. `interval` is "<n> <PERIOD>" ("1 YEAR",
|
||||
"1 HOUR"); the last word, lowered, is the period. With nothing to build
|
||||
a line from, Ashby's own prose summary (`compensationTierSummary`, else
|
||||
`scrapeableCompensationSalarySummary`) is what is left to show."""
|
||||
components = compensation.get("summaryComponents") or []
|
||||
stated = [c for c in components if num(c.get("minValue")) is not None
|
||||
or num(c.get("maxValue")) is not None]
|
||||
lines = []
|
||||
for component in stated:
|
||||
period = str(component.get("interval") or "").split(" ")[-1].lower()
|
||||
line = pay_line(num(component.get("minValue")), num(component.get("maxValue")),
|
||||
component.get("currencyCode"), period)
|
||||
if not line:
|
||||
continue
|
||||
if len(stated) > 1:
|
||||
label = str(component.get("compensationType") or "").strip()
|
||||
line = f"{label}: {line}" if label else line
|
||||
lines.append(line)
|
||||
if lines:
|
||||
return "; ".join(lines)
|
||||
return str(compensation.get("compensationTierSummary")
|
||||
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("jobs") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
||||
compensation = j.get("compensation") or {}
|
||||
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (j.get("location") or "").strip(),
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("publishedAt")),
|
||||
"department": (j.get("department") or j.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": j.get("isRemote"),
|
||||
"salary_text": _salary_text(compensation),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Greenhouse: `greenhouse:<board token>`, the whole board in one JSON answer.
|
||||
|
||||
The field map is ported from `openings.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line
|
||||
|
||||
NAME = "greenhouse"
|
||||
LABEL = "Greenhouse"
|
||||
RANK = 10
|
||||
PATTERNS = (
|
||||
r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)",
|
||||
r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)",
|
||||
)
|
||||
|
||||
_API = "boards-api.greenhouse.io"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return path_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://job-boards.greenhouse.io/{slug}"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{_API}/v1/boards/{slug}/jobs?content=true&pay_transparency=true"
|
||||
|
||||
|
||||
def fallback_url(slug: str) -> str:
|
||||
"""The feed without `content=true`, read once when the full one exceeds
|
||||
the byte cap: the descriptions are lost, but the titles land rather than
|
||||
the whole board being skipped."""
|
||||
return f"https://{_API}/v1/boards/{slug}/jobs"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return f"https://{_API}/v1/boards/{slug}"
|
||||
|
||||
|
||||
def _salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
||||
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
|
||||
on the feed URL, see `feed_url`), which the list endpoint already carries
|
||||
in the same request — no per-role fetch. Only when there is exactly one
|
||||
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
|
||||
one per office) gives no way to tell which office the role in hand is
|
||||
at, and picking one would misrepresent the columns. Greenhouse's own
|
||||
ranges are cents, and yearly by convention except an hourly or a daily
|
||||
one, named as such in `title` with no separate field to check — those
|
||||
are skipped rather than annualised into a number nobody stated."""
|
||||
if len(ranges) != 1:
|
||||
return None, None, ""
|
||||
entry = ranges[0]
|
||||
title = str(entry.get("title") or "").lower()
|
||||
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
|
||||
return None, None, ""
|
||||
code = str(entry.get("currency_type") or "").strip().upper()
|
||||
if not code:
|
||||
return None, None, ""
|
||||
lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents"))
|
||||
if lo is None and hi is None:
|
||||
return None, None, ""
|
||||
return (int(lo / 100) if lo is not None else None,
|
||||
int(hi / 100) if hi is not None else None, code)
|
||||
|
||||
|
||||
def _salary_text(ranges: list[dict[str, Any]]) -> str:
|
||||
"""One line per `pay_input_ranges` entry, joined with "; " — every range
|
||||
Greenhouse states, not only the single one `_salary` accepts for the
|
||||
columns. With more than one, each is labelled with the board's own
|
||||
`title` ("Zone 1 Pay Range: ..."), since that is the only way to tell the
|
||||
ranges apart; with just the one, the label adds nothing and is left off.
|
||||
The hour/day check is the same one `_salary` uses to keep those out of
|
||||
the yearly columns; here it only picks the word that follows "per"."""
|
||||
multi = len(ranges) > 1
|
||||
lines = []
|
||||
for entry in ranges:
|
||||
title = str(entry.get("title") or "").strip()
|
||||
low_title = title.lower()
|
||||
period = ("hour" if any(w in low_title for w in ("hour", "/hr"))
|
||||
else "day" if "day" in low_title else "year")
|
||||
lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents"))
|
||||
line = pay_line(lo / 100 if lo is not None else None,
|
||||
hi / 100 if hi is not None else None,
|
||||
entry.get("currency_type"), period)
|
||||
if not line:
|
||||
continue
|
||||
if multi:
|
||||
label = title.rstrip(":").strip() or "Range"
|
||||
line = f"{label}: {line}"
|
||||
lines.append(line)
|
||||
return "; ".join(lines)
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("jobs") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("absolute_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
departments = j.get("departments") or []
|
||||
# Greenhouse ships the description HTML-entity-escaped inside the
|
||||
# JSON string (`<p>`), so it needs unescaping before the tags
|
||||
# themselves can be stripped.
|
||||
content = html.unescape(j.get("content") or "")
|
||||
ranges = j.get("pay_input_ranges") or []
|
||||
salary_min, salary_max, currency = _salary(ranges)
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": ((j.get("location") or {}).get("name") or "").strip(),
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("first_published") or j.get("updated_at")),
|
||||
"department": (departments[0].get("name") if departments else "") or "",
|
||||
"description": strip_html(content),
|
||||
"remote": None, # Greenhouse's feed does not say
|
||||
"salary_text": _salary_text(ranges),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Lever: `lever:<site>`, or `lever:eu/<site>` for a board on Lever's EU
|
||||
instance, the whole board in one JSON answer.
|
||||
|
||||
Lever runs two instances that share nothing but the API: `api.lever.co` and
|
||||
`api.eu.lever.co`, with the pages on `jobs.lever.co` and `jobs.eu.lever.co`.
|
||||
A site exists on one of them only (TomTom and Prima answer 404 on the other,
|
||||
2026-09-29), and the same name on both would be two companies, so the region
|
||||
is part of the slug. It is a prefix a plain site name cannot carry, since a
|
||||
Lever site name has no slash, which leaves every `lever:<site>` target as it
|
||||
was.
|
||||
|
||||
The field map is ported from `openings.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
|
||||
|
||||
NAME = "lever"
|
||||
LABEL = "Lever"
|
||||
RANK = 20
|
||||
#: Two groups on the EU hosts, `eu` and the site, which discovery joins into
|
||||
#: the slug `eu/<site>`.
|
||||
PATTERNS = (
|
||||
r"jobs\.lever\.co/([A-Za-z0-9_-]+)",
|
||||
r"api\.lever\.co/v0/postings/([A-Za-z0-9_-]+)",
|
||||
r"jobs\.(eu)\.lever\.co/([A-Za-z0-9_-]+)",
|
||||
r"api\.(eu)\.lever\.co/v0/postings/([A-Za-z0-9_-]+)",
|
||||
)
|
||||
|
||||
#: The region prefixes a slug may carry, to the infix of both host names.
|
||||
_REGIONS = {"eu": "eu."}
|
||||
|
||||
|
||||
def _split(slug: str) -> tuple[str, str]:
|
||||
"""`(host infix, site)`: `("eu.", "tomtom")` for `eu/tomtom`, `("", "acme")`
|
||||
for `acme`."""
|
||||
region, sep, site = slug.partition("/")
|
||||
return (_REGIONS.get(region, "?"), site) if sep else ("", slug)
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
infix, site = _split(slug)
|
||||
return infix != "?" and path_slug_ok(site)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(f"api.{_split(slug)[0]}lever.co")
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
infix, site = _split(slug)
|
||||
return f"https://jobs.{infix}lever.co/{site}"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
infix, site = _split(slug)
|
||||
return f"https://api.{infix}lever.co/v0/postings/{site}?mode=json"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
infix, site = _split(slug)
|
||||
return f"https://api.{infix}lever.co/v0/postings/{site}?limit=1&mode=json"
|
||||
|
||||
|
||||
def probe_slugs(guess: str) -> tuple[str, ...]:
|
||||
"""A name is tried on both instances, the US one first, since a site that
|
||||
names neither leaves no other way to tell which one it is on."""
|
||||
return (guess, *(f"{region}/{guess}" for region in _REGIONS))
|
||||
|
||||
|
||||
def _salary_text(salary_range: dict[str, Any]) -> str:
|
||||
"""One line from `salaryRange`, whatever its `interval` — including an
|
||||
hourly or daily one, which `yearly_amount` drops from the columns.
|
||||
Lever's interval is "per-<period>-<kind>" ("per-year-salary",
|
||||
"per-hour-wage"); the middle word is the period."""
|
||||
interval = str(salary_range.get("interval") or "")
|
||||
parts = interval.split("-")
|
||||
period = parts[1] if len(parts) >= 3 else ""
|
||||
return pay_line(num(salary_range.get("min")), num(salary_range.get("max")),
|
||||
salary_range.get("currency"), period)
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
rows = []
|
||||
for j in data:
|
||||
title = (j.get("text") or "").strip()
|
||||
url = (j.get("hostedUrl") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
categories = j.get("categories") or {}
|
||||
description = j.get("description") or j.get("descriptionPlain") or ""
|
||||
salary_range = j.get("salaryRange") or {}
|
||||
salary_min, salary_max, currency = yearly_amount(
|
||||
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
|
||||
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (categories.get("location") or "").strip(),
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("createdAt")),
|
||||
"department": (categories.get("department") or categories.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": (j.get("workplaceType") or "").strip() or None,
|
||||
"salary_text": _salary_text(salary_range),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Personio: `personio:<subdomain>`, the whole board in one XML answer from
|
||||
the company's own subdomain of jobs.personio.de.
|
||||
|
||||
Written from one real board on 2026-09-23; jobhunt-hq did not read
|
||||
Personio.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
NAME = "personio"
|
||||
LABEL = "Personio"
|
||||
RANK = 70
|
||||
PATTERNS = (r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)",)
|
||||
|
||||
#: Personio's `salaryInformation/type` spellings, to the plain period word
|
||||
#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through.
|
||||
_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
|
||||
"daily": "day", "weekly": "week"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return label_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
"""The company's own subdomain, so one company's slug cannot pass for
|
||||
another's."""
|
||||
return only(f"{slug}.jobs.personio.de")
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{slug}.jobs.personio.de"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{slug}.jobs.personio.de/xml"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return feed_url(slug)
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
return b"<position>" in body
|
||||
|
||||
|
||||
def _salary_text(salary_info: ET.Element) -> str:
|
||||
"""One line from a `<salaryInformation>` element, whatever its `type`,
|
||||
including an hourly or daily one that `yearly_amount` drops from the
|
||||
columns."""
|
||||
period_raw = (salary_info.findtext("type") or "").strip().lower()
|
||||
period = _PERIODS.get(period_raw, period_raw)
|
||||
return pay_line(num(salary_info.findtext("min")), num(salary_info.findtext("max")),
|
||||
salary_info.findtext("currencyCode"), period)
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""`{slug}.jobs.personio.de/xml` -> `<workzag-jobs><position>...`. No job
|
||||
URL in the feed itself; Personio's own page is `/job/{id}`, confirmed
|
||||
live against a real posting (2026-09-23)."""
|
||||
root = ET.fromstring(body)
|
||||
rows = []
|
||||
for position in root.findall("position"):
|
||||
job_id = (position.findtext("id") or "").strip()
|
||||
title = (position.findtext("name") or "").strip()
|
||||
if not job_id or not title:
|
||||
continue
|
||||
description = " ".join(
|
||||
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
||||
salary_info = position.find("salaryInformation")
|
||||
salary_min = salary_max = None
|
||||
currency = ""
|
||||
salary_text = ""
|
||||
if salary_info is not None:
|
||||
salary_min, salary_max, currency = yearly_amount(
|
||||
salary_info.findtext("min"), salary_info.findtext("max"),
|
||||
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
|
||||
{"yearly": 1, "monthly": 12})
|
||||
salary_text = _salary_text(salary_info)
|
||||
rows.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"location": (position.findtext("office") or "").strip(),
|
||||
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
|
||||
"posted": parse_dt(position.findtext("createdAt")),
|
||||
"department": (position.findtext("department") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": "remote" in (position.findtext("office") or "").lower() or None,
|
||||
"salary_text": salary_text,
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Recruitee: `recruitee:<subdomain>`, the whole board in one JSON answer
|
||||
from the company's own subdomain of recruitee.com.
|
||||
|
||||
Written from one real board on 2026-09-23; jobhunt-hq did not read
|
||||
Recruitee.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
NAME = "recruitee"
|
||||
LABEL = "Recruitee"
|
||||
RANK = 60
|
||||
PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",)
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return label_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
"""The company's own subdomain, so one company's slug cannot pass for
|
||||
another's."""
|
||||
return only(f"{slug}.recruitee.com")
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{slug}.recruitee.com"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{slug}.recruitee.com/api/offers/"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return feed_url(slug)
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
return b'"offers":[' in body and b'"offers":[]' not in body
|
||||
|
||||
|
||||
def _salary_text(salary: dict[str, Any]) -> str:
|
||||
"""One line from Recruitee's `salary` object; `period` is already a
|
||||
plain word ("year", "month")."""
|
||||
period = str(salary.get("period") or "").strip().lower()
|
||||
return pay_line(num(salary.get("min")), num(salary.get("max")),
|
||||
salary.get("currency"), period)
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("offers") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("careers_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
||||
salary_min, salary_max, currency = yearly_amount(
|
||||
salary.get("min"), salary.get("max"), salary.get("currency"),
|
||||
salary.get("period"), {"year": 1, "month": 12})
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (j.get("location") or j.get("city") or "").strip(),
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("published_at") or j.get("created_at")),
|
||||
"department": (j.get("department") or "").strip(),
|
||||
"description": strip_html(j.get("description") or ""),
|
||||
"remote": j.get("remote"),
|
||||
"salary_text": _salary_text(salary),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,90 @@
|
||||
"""SmartRecruiters: `smartrecruiters:<company identifier>`, paged by offset
|
||||
with a stated total. The identifier is case-sensitive (`BoschGroup`).
|
||||
|
||||
Written from one real board on 2026-09-23; jobhunt-hq did not read
|
||||
SmartRecruiters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
|
||||
NAME = "smartrecruiters"
|
||||
LABEL = "SmartRecruiters"
|
||||
RANK = 50
|
||||
PATTERNS = (
|
||||
r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)",
|
||||
)
|
||||
|
||||
_API = "api.smartrecruiters.com"
|
||||
|
||||
#: The page size asked for, which is also the most the host serves per page.
|
||||
PAGE_SIZE = 100
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return path_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://jobs.smartrecruiters.com/{slug}"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return f"https://{_API}/v1/companies/{slug}/postings?limit=1"
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
"""An unknown identifier answers an empty listing, so a listing without
|
||||
postings counts as no board."""
|
||||
return b'"totalFound":' in body and b'"totalFound":0' not in body
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings"
|
||||
f"?offset={offset}&limit={PAGE_SIZE}"))
|
||||
content = data.get("content") or []
|
||||
return data, content, data.get("totalFound"), offset + len(content) if content else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
|
||||
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
|
||||
page, whatever `limit` asks for; the next is `?offset=` further on. The
|
||||
list omits the job description (only the per-posting detail endpoint has
|
||||
it, and fetching one per row would be hundreds of extra requests per
|
||||
board), so `description` stays empty here."""
|
||||
out = []
|
||||
for j in data.get("content") or []:
|
||||
title = (j.get("name") or "").strip()
|
||||
job_id = str(j.get("id") or "")
|
||||
if not title or not job_id:
|
||||
continue
|
||||
location = j.get("location") or {}
|
||||
loc_text = (location.get("fullLocation")
|
||||
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
||||
out.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"location": (loc_text or "").strip(),
|
||||
"url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}",
|
||||
"posted": parse_dt(j.get("releasedDate")),
|
||||
"department": ((j.get("department") or {}).get("label") or "").strip(),
|
||||
"description": "",
|
||||
"remote": location.get("remote"),
|
||||
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Teamtailor: `teamtailor:<subdomain>` for acme.teamtailor.com, or, for a
|
||||
career site on the company's own domain, that whole host name:
|
||||
`teamtailor:careers.acme.com`. An RSS feed paged by offset that states no
|
||||
total.
|
||||
|
||||
Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "teamtailor"
|
||||
LABEL = "Teamtailor"
|
||||
RANK = 80
|
||||
PATTERNS = (
|
||||
r"https?://([a-z0-9-]+)\.teamtailor\.com",
|
||||
# No slug: a career site on the company's own domain, which only loads
|
||||
# Teamtailor's CDN. Discovery reads its board off the page's host.
|
||||
r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets",
|
||||
)
|
||||
|
||||
#: The page size, asked for as `per_page`. A feed with neither parameter
|
||||
#: answers with every role (73 of 73 at Shine, 2026-09-29), but says nowhere
|
||||
#: that it is the whole list, so this reads by page to an empty one;
|
||||
#: `per_page=200` was answered in full too, and the cursor advances by what a
|
||||
#: page returned, so a smaller cap on the host's side would not skip a role.
|
||||
PAGE_SIZE = 50
|
||||
|
||||
#: Only a DNS name with a letter in its last label is a site, so an address
|
||||
#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could
|
||||
#: be pointed at; `ats._get` still refuses every host but this one, redirects
|
||||
#: included.
|
||||
_HOST = re.compile(r"[a-z0-9-]+(\.[a-z0-9-]+)*")
|
||||
|
||||
#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`.
|
||||
_TT = "{https://teamtailor.com/locations}"
|
||||
|
||||
#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing
|
||||
#: about where the role is worked and `temporary` is remote for a spell only,
|
||||
#: so both stay unknown rather than become "onsite" or "remote".
|
||||
_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"}
|
||||
|
||||
|
||||
def site(slug: str) -> str:
|
||||
"""The host a career site is served from: `acme` is acme.teamtailor.com,
|
||||
and a slug with a dot is a site on the company's own domain,
|
||||
`careers.acme.com`, taken whole (the feed path is the same on both)."""
|
||||
slug = slug.strip().lower()
|
||||
return slug if "." in slug else f"{slug}.teamtailor.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
host = site(slug)
|
||||
return bool(slug) and slug == slug.strip() and bool(_HOST.fullmatch(host)) \
|
||||
and not host.rsplit(".", 1)[-1].isdigit()
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(site(slug))
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{site(slug)}/jobs"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return f"https://{site(slug)}/jobs.rss"
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
return b"<item>" in body
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
items = _items(get(f"https://{site(slug)}/jobs.rss?per_page={PAGE_SIZE}&offset={offset}"))
|
||||
results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items]
|
||||
return items, results, None, offset + len(items) if items else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(_items(body), slug)
|
||||
|
||||
|
||||
def _items(body: bytes) -> list[ET.Element]:
|
||||
"""`{site}/jobs.rss` -> `<rss><channel><item>...`, the feed every Teamtailor
|
||||
career site serves (checked live on a dozen sites, 2026-09-29). A missing
|
||||
`channel` is a page that is not a feed, and raises."""
|
||||
channel = ET.fromstring(body).find("channel")
|
||||
if channel is None:
|
||||
raise ValueError("not a Teamtailor feed: no channel")
|
||||
return channel.findall("item")
|
||||
|
||||
|
||||
def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
"""One page of items. The item has the title, the job URL (on the site's
|
||||
own domain), a `guid` uuid, `pubDate`, the HTML `description` in full,
|
||||
`remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the
|
||||
`tt:` namespace, `locations` (a list: `city` and `country` each, `name` free
|
||||
text), `department` and `role`, either of which may be empty. `place` joins
|
||||
the distinct locations with "; ", which `country_of` reads a location at a
|
||||
time. No pay is stated anywhere in the feed."""
|
||||
out = []
|
||||
for item in items:
|
||||
title = (item.findtext("title") or "").strip()
|
||||
url = (item.findtext("link") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
places: list[str] = []
|
||||
for loc in item.findall(f"{_TT}locations/{_TT}location"):
|
||||
city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip()
|
||||
country = (loc.findtext(f"{_TT}country") or "").strip()
|
||||
place = city if country == city else ", ".join(p for p in (city, country) if p)
|
||||
if place and place not in places:
|
||||
places.append(place)
|
||||
out.append({
|
||||
"source_id": (item.findtext("guid") or url).strip(),
|
||||
"title": title,
|
||||
"location": "; ".join(places),
|
||||
"url": url,
|
||||
"posted": parse_dt(item.findtext("pubDate")),
|
||||
"department": ((item.findtext(f"{_TT}department") or "").strip()
|
||||
or (item.findtext(f"{_TT}role") or "").strip()),
|
||||
"description": html.unescape(strip_html(item.findtext("description") or "")),
|
||||
"remote": _REMOTE.get((item.findtext("remoteStatus") or "").strip()),
|
||||
"salary_text": "", # Teamtailor's feed states no pay at all
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Workable: `workable:<account>`, paged ten roles at a time with a stated
|
||||
total.
|
||||
|
||||
Written from one real board, Hugging Face's, on 2026-09-28, when the first
|
||||
board with roles was seen; jobhunt-hq did not read Workable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
|
||||
NAME = "workable"
|
||||
LABEL = "Workable"
|
||||
RANK = 40
|
||||
PATTERNS = (
|
||||
r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)",
|
||||
r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com",
|
||||
)
|
||||
|
||||
_API = "apply.workable.com"
|
||||
|
||||
# No probe: Workable's API sits behind a rate limit that a run of guesses
|
||||
# trips, and a Workable site links its board plainly anyway.
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return path_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_API}/{slug}"
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""One POST of the list; the next page is the same POST with
|
||||
`{"token": nextPage}`, and the last page has no `nextPage`."""
|
||||
url = f"https://{_API}/api/v3/accounts/{slug}/jobs"
|
||||
data = json.loads(get(url, method="POST", payload={"token": cursor} if cursor else {}))
|
||||
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
|
||||
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
|
||||
Face's board answered on 2026-09-28. A result carries no URL and no text:
|
||||
the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and
|
||||
the description is only on the per-job endpoint, one request per role,
|
||||
so `description` stays empty here as it does for SmartRecruiters.
|
||||
`location` is an object and `department` a list; `remote` is a flag
|
||||
beside `workplace` (remote, hybrid, on_site)."""
|
||||
out = []
|
||||
for j in data.get("results") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
shortcode = (j.get("shortcode") or "").strip()
|
||||
if not title or not shortcode:
|
||||
continue
|
||||
loc = j.get("location") or {}
|
||||
location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p)
|
||||
dept = j.get("department") or ""
|
||||
if isinstance(dept, list):
|
||||
dept = ", ".join(d for d in dept if d)
|
||||
out.append({
|
||||
"source_id": str(j.get("id") or shortcode),
|
||||
"title": title,
|
||||
"location": location,
|
||||
"url": f"https://{_API}/{slug}/j/{shortcode}/",
|
||||
"posted": parse_dt(j.get("published")),
|
||||
"department": dept.strip(),
|
||||
"description": "",
|
||||
"remote": bool(j.get("remote")) or j.get("workplace") == "remote",
|
||||
"salary_text": "", # Workable's list endpoint states no pay at all
|
||||
})
|
||||
return out
|
||||
+6
-5
@@ -19,6 +19,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.atshosts import smartrecruiters # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -384,7 +385,7 @@ def test_is_known_target_accepts_the_eight_hosts_and_refuses_others() -> None:
|
||||
"recruitee", "personio", "teamtailor"):
|
||||
assert ats.is_known_target(f"{host}:acme")
|
||||
assert not ats.is_known_target("linkedin:acme")
|
||||
assert not ats.is_known_target("workday:acme")
|
||||
assert not ats.is_known_target("taleo:acme")
|
||||
assert not ats.is_known_target("greenhouse:")
|
||||
assert not ats.is_known_target("nohostnoseparator")
|
||||
# a Teamtailor slug names a host: its own subdomain, or a whole domain
|
||||
@@ -420,7 +421,7 @@ def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
|
||||
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
|
||||
assert ats.careers_url("teamtailor:acme") == "https://acme.teamtailor.com/jobs"
|
||||
assert ats.careers_url("teamtailor:careers.acme.com") == "https://careers.acme.com/jobs"
|
||||
assert ats.careers_url("workday:acme") == ""
|
||||
assert ats.careers_url("taleo:acme") == ""
|
||||
|
||||
|
||||
# --------------------------------------------------------------- body cap
|
||||
@@ -570,7 +571,7 @@ def test_smartrecruiters_pages_by_offset_to_total_found(
|
||||
urls.append(url)
|
||||
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
|
||||
@@ -588,7 +589,7 @@ def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
return empty if "offset=2" in url else _page("smartrecruiters.json")
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
assert (len(rows), complete) == (2, False)
|
||||
@@ -604,7 +605,7 @@ def test_a_repeated_row_does_not_count_twice_toward_the_total(
|
||||
page["content"] = page["content"][1:]
|
||||
return json.dumps(page).encode()
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
assert (len(rows), complete) == (2, False)
|
||||
|
||||
@@ -0,0 +1,761 @@
|
||||
"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for
|
||||
every host: the registry and its contract check, the fetch a host is handed,
|
||||
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats, atshosts # noqa: E402
|
||||
from plugins.atshosts import lever # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
|
||||
# --------------------------------------------------------------- registry
|
||||
|
||||
def test_the_registry_finds_every_module() -> None:
|
||||
names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py")
|
||||
if not p.stem.startswith("_"))
|
||||
assert sorted(atshosts.HOSTS) == names
|
||||
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||
"recruitee", "personio", "teamtailor"}
|
||||
assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items())
|
||||
# The Sources page's select and the plugin's description follow the modules.
|
||||
assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS
|
||||
assert "Teamtailor" in ats.ATSSource.description
|
||||
|
||||
|
||||
def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str,
|
||||
modules: dict[str, str]) -> list[str]:
|
||||
package = tmp_path / name
|
||||
package.mkdir()
|
||||
(package / "__init__.py").write_text("")
|
||||
for module, source in modules.items():
|
||||
(package / f"{module}.py").write_text(source)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
return [str(package)]
|
||||
|
||||
|
||||
_GOOD = """
|
||||
NAME = "{name}"
|
||||
LABEL = "Good"
|
||||
PATTERNS = (r"good\\.example/([a-z]+)",)
|
||||
def slug_ok(slug): return True
|
||||
def allowed(slug): return lambda h: h == "good.example"
|
||||
def careers_url(slug): return "https://good.example/" + slug
|
||||
def feed_url(slug): return "https://good.example/" + slug + ".json"
|
||||
def parse(body, slug): return []
|
||||
"""
|
||||
|
||||
|
||||
def test_the_registry_loads_ats_by_path_without_the_repo_root_on_the_path(
|
||||
tmp_path: Path) -> None:
|
||||
"""As `desk.sources.registry` does in a process the app did not set up:
|
||||
another working directory, and `plugins` not importable at first."""
|
||||
code = (
|
||||
"import importlib.util, sys\n"
|
||||
"assert importlib.util.find_spec('plugins') is None, 'plugins is importable'\n"
|
||||
"from desk.sources import registry\n"
|
||||
f"print(sorted(registry.available([{str(ROOT / 'plugins')!r}])))\n")
|
||||
env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"}
|
||||
done = subprocess.run([sys.executable, "-c", code], cwd=tmp_path, env=env,
|
||||
capture_output=True, text=True, timeout=60, check=False)
|
||||
assert done.returncode == 0, done.stderr
|
||||
assert "'ats'" in done.stdout and "'search'" in done.stdout
|
||||
|
||||
|
||||
def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
path = _package(tmp_path, monkeypatch, "hostsok", {
|
||||
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
|
||||
"_helpers": "X = 1\n"})
|
||||
assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"]
|
||||
|
||||
|
||||
def test_the_registry_orders_by_rank_then_name(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
path = _package(tmp_path, monkeypatch, "hostsranked", {
|
||||
"alpha": _GOOD.format(name="alpha"), # the default, 100
|
||||
"beta": _GOOD.format(name="beta") + "RANK = 5\n",
|
||||
"gamma": _GOOD.format(name="gamma") + "RANK = 100\n"})
|
||||
assert list(atshosts.load(path, "hostsranked")) == ["beta", "alpha", "gamma"]
|
||||
|
||||
|
||||
def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
|
||||
"""The Sources page's select opens on Greenhouse, which its hint names,
|
||||
and a guessed slug on two hosts goes to the one probed first, as before."""
|
||||
first = ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
|
||||
"personio", "teamtailor"]
|
||||
assert list(ats.HOSTS)[:8] == first
|
||||
assert ats.ATSSource.description.startswith(
|
||||
"Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, "
|
||||
"Recruitee, Personio, Teamtailor")
|
||||
assert all(module.RANK < atshosts.DEFAULT_RANK
|
||||
for name, module in atshosts.HOSTS.items() if name in first)
|
||||
|
||||
|
||||
def test_the_registry_refuses_a_module_missing_a_required_name(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
|
||||
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
|
||||
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
|
||||
atshosts.load(path, "hostsbad")
|
||||
|
||||
|
||||
def test_the_contract_check_names_each_problem() -> None:
|
||||
module: Any = types.ModuleType("odd")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",)
|
||||
module.slug_ok = module.allowed = module.careers_url = lambda slug: ""
|
||||
module.page = lambda get, slug, cursor: None
|
||||
module.probe_ok = lambda body: True
|
||||
module.RANK, module.KEEP_CASE = "1", "yes"
|
||||
found = atshosts.problems(module, "odd")
|
||||
assert "NAME is 'other', not the module's name 'odd'" in found
|
||||
assert "LABEL is not a non-empty string" in found
|
||||
assert any(p.startswith("PATTERNS has '('") for p in found)
|
||||
assert "page without rows" in found
|
||||
assert "probe_ok without probe_url" in found
|
||||
assert "RANK is not an int" in found and "KEEP_CASE is not a bool" in found
|
||||
|
||||
|
||||
@pytest.mark.parametrize("host", ["greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||
"recruitee", "personio", "teamtailor"])
|
||||
def test_a_slug_is_matched_whole(host: str) -> None:
|
||||
assert ats.is_known_target(f"{host}:acme")
|
||||
for slug in ("acme\n", "acme\r\n", " acme", "acme ", "ac me", "acme/x", "acme?x", "acme#x"):
|
||||
assert not ats.is_known_target(f"{host}:{slug}"), slug
|
||||
|
||||
|
||||
def test_a_slug_in_the_host_name_is_allowed_whatever_its_case() -> None:
|
||||
for host, api in (("recruitee", "acme.recruitee.com"), ("personio", "acme.jobs.personio.de")):
|
||||
allow = ats._allowed_host(host, "Acme")
|
||||
assert allow(api) and allow(api.upper())
|
||||
assert not allow(api.replace("acme", "other"))
|
||||
|
||||
|
||||
# ------------------------------------------------------------ the fetch
|
||||
|
||||
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def test_get_sends_extra_headers_on_top_of_its_own() -> None:
|
||||
seen: list[httpx.Headers] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request.headers)
|
||||
return httpx.Response(200, content=b"ok")
|
||||
|
||||
with _client(handler) as client:
|
||||
get = ats._getter(client, "lever", "acme")
|
||||
assert get("https://api.lever.co/v0/postings/acme",
|
||||
headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok"
|
||||
get("https://api.lever.co/v0/postings/acme")
|
||||
assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme"
|
||||
assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT
|
||||
assert seen[1]["accept"] == "application/json, text/xml"
|
||||
assert "x-board" not in seen[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("site", [
|
||||
"metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa",
|
||||
"app.localhost", "foo.0x7f000001", "printer.local.",
|
||||
])
|
||||
def test_get_refuses_a_local_name_before_any_request(site: str) -> None:
|
||||
"""A Teamtailor slug may be any host name, so the allow-list admits
|
||||
these; `_get` refuses them before a request is made."""
|
||||
asked: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
asked.append(str(request.url))
|
||||
return httpx.Response(200, content=b"<rss><channel/></rss>")
|
||||
|
||||
with _client(handler) as client:
|
||||
# A trailing dot is no Teamtailor slug at all; the rest are, and are refused.
|
||||
with pytest.raises(ValueError if site.endswith(".") else ats._Refused):
|
||||
ats._fetch_rows(client, f"teamtailor:{site}")
|
||||
for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/",
|
||||
"https://2130706433/", "https://127.1/", "https://localhost/"):
|
||||
with pytest.raises(ats._Refused):
|
||||
ats._get(client, url, "teamtailor", site)
|
||||
assert asked == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", [
|
||||
"https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x",
|
||||
"https://localhost/x",
|
||||
])
|
||||
def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(302, headers={"location": location})
|
||||
|
||||
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
|
||||
|
||||
def test_get_takes_port_443_written_out_and_refuses_another() -> None:
|
||||
with _client(lambda request: httpx.Response(200, content=b"[]")) as client:
|
||||
assert ats._get(client, "https://api.lever.co:443/v0/postings/acme",
|
||||
"lever", "acme") == b"[]"
|
||||
with pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme")
|
||||
|
||||
|
||||
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Bound before the replacement, and still reaching it: `_get` is looked
|
||||
up when a request is made."""
|
||||
client = object()
|
||||
get = ats._getter(client, "lever", "acme") # type: ignore[arg-type]
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def replaced(*args: Any, **kwargs: Any) -> bytes:
|
||||
calls.append((args, kwargs))
|
||||
return b""
|
||||
|
||||
monkeypatch.setattr(ats, "_get", replaced)
|
||||
get("https://api.lever.co/v0/postings/acme", headers={"X": "1"})
|
||||
get("https://api.lever.co/v0/postings/acme", method="POST", payload={})
|
||||
assert calls == [
|
||||
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
||||
{"method": "GET", "headers": {"X": "1"}}),
|
||||
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
||||
{"method": "POST", "payload": {}})]
|
||||
|
||||
|
||||
# ------------------------------------------------------- a fake host
|
||||
|
||||
def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||
robots: bool = False) -> types.ModuleType:
|
||||
"""A host `fake` whose feed is a JSON list of ids, each role with a
|
||||
teaser for a description, and, with `detail`, a per-role page; with
|
||||
`robots`, its listing is the page robots.txt must allow."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
module.parse = lambda body, slug: [
|
||||
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
||||
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
||||
"description": "teaser", "remote": None, "salary_text": ""}
|
||||
for i in json.loads(body)]
|
||||
if detail is not None:
|
||||
module.detail = detail
|
||||
if robots:
|
||||
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
return module
|
||||
|
||||
|
||||
def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError:
|
||||
request = httpx.Request("GET", url)
|
||||
return httpx.HTTPStatusError(str(code), request=request,
|
||||
response=httpx.Response(code, request=request))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||
"""`ats._get` answering from a dict of URL -> body or exception, and
|
||||
remembering what it was asked. No delay between requests."""
|
||||
answers: dict[str, Any] = {}
|
||||
asked: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
||||
www_twin=False):
|
||||
asked.append(url)
|
||||
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
||||
raise ats._Refused(url)
|
||||
answer = answers.get(url, _status(404, url))
|
||||
if isinstance(answer, BaseException):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
return {"answers": answers, "asked": asked}
|
||||
|
||||
|
||||
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
||||
state: dict[str, str] | None = None) -> tuple[list[Any], Any]:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", module)
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||
if state is not None:
|
||||
source.state = state
|
||||
return list(source.fetch()), source.state
|
||||
|
||||
|
||||
def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The fake host's detail: the role's own page is its description."""
|
||||
return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(),
|
||||
"location": "Copenhagen"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
def test_detail_is_read_for_unread_roles_only(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
|
||||
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
|
||||
module = _fake_host(detail=_detail)
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
|
||||
assert [(i.source_id, i.body) for i in items] == [
|
||||
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
|
||||
# What detail returns replaces the listing's field; a role not read keeps the listing's.
|
||||
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
ids = [str(n) for n in range(5)]
|
||||
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
|
||||
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
|
||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||
module = _fake_host(detail=_detail)
|
||||
items, state = _run(monkeypatch, module)
|
||||
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
|
||||
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
|
||||
|
||||
|
||||
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert [(i.source_id, i.body) for i in items] == [
|
||||
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
|
||||
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||
f"{base}/2": _status(429), f"{base}/3": b"three"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_ids_no_longer_listed_leave_the_detailed_list(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""A listing cut short says nothing about the roles it left out, so their
|
||||
ids stay, and the next pull does not read them in detail again."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
||||
atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
|
||||
|
||||
def test_fetch_board_reads_the_listing_only(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
||||
rows, complete = ats.fetch_board("fake:acme")
|
||||
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
|
||||
assert web["asked"] == [f"{base}/list"]
|
||||
|
||||
|
||||
def test_a_host_without_detail_keeps_the_listings_description(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]'
|
||||
items, state = _run(monkeypatch, _fake_host())
|
||||
assert [i.body for i in items] == ["teaser"]
|
||||
assert "ats:detailed:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_detail_answering_404_or_410_counts_as_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
|
||||
f"{base}/3": b"three"}) # 1 answers 404
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert [i.body for i in items] == ["", "", "three"]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
|
||||
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
"""Two roles that always fail and a cap of two: the third is read on the
|
||||
second pull, since roles never tried go before those that failed."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
|
||||
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
|
||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||
module = _fake_host(detail=_detail)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
|
||||
assert [i.body for i in items] == ["", "", "three", "four"]
|
||||
|
||||
# With nothing new left, the failed ones are tried again, within the cap.
|
||||
web["asked"].clear()
|
||||
web["answers"][f"{base}/1"] = b"one at last"
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one at last", "", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == []
|
||||
|
||||
|
||||
def test_failed_ids_no_longer_listed_leave_the_failed_list(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
|
||||
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any],
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""`salary_text`, `department` and `title` would be rewritten from the
|
||||
listing on the next pull, so detail does not fill them."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]'})
|
||||
|
||||
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
|
||||
"salary_min": 60000, "salary_max": 70000,
|
||||
"posted": datetime(2026, 9, 1, tzinfo=UTC),
|
||||
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
|
||||
"title": "Another title"}
|
||||
|
||||
caplog.set_level("DEBUG", logger=ats.log.name)
|
||||
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
||||
assert (item.body, item.title, item.published_at) == (
|
||||
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
||||
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
|
||||
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
|
||||
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
|
||||
assert "department, salary_text, title left out" in caplog.text
|
||||
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
|
||||
"salary_min", "salary_max", "posted"}
|
||||
|
||||
|
||||
def _paged_host() -> types.ModuleType:
|
||||
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
|
||||
module: Any = _fake_host(detail=_detail)
|
||||
del module.feed_url, module.parse
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
|
||||
index = int(cursor or 0)
|
||||
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
|
||||
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
|
||||
|
||||
module.page = page
|
||||
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
|
||||
assert not atshosts.problems(module, "fake")
|
||||
return module
|
||||
|
||||
|
||||
def test_a_paged_host_reads_every_page_then_each_role_once(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
|
||||
f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||
items, state = _run(monkeypatch, _paged_host())
|
||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
|
||||
f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one", "two"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, _paged_host(), state)
|
||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
|
||||
assert [i.body for i in items] == ["", ""]
|
||||
|
||||
|
||||
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
|
||||
f"{base}/1": b"one"})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
||||
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
|
||||
_items, state = _run(monkeypatch, _paged_host(), state)
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
|
||||
|
||||
|
||||
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The consumer stops after the first role, so the second, read but not
|
||||
stored, is read again on the next pull."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||
roles: Any = source.fetch() # a generator, which `close` stops
|
||||
assert next(roles).body == "one"
|
||||
roles.close()
|
||||
assert "ats:detailed:fake:acme" not in source.state
|
||||
|
||||
|
||||
# ------------------------------------------------------------ robots.txt
|
||||
|
||||
ROBOTS = "https://jobs.fake.example/robots.txt"
|
||||
|
||||
|
||||
def test_robots_that_allow_the_page_let_the_board_be_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("robots", [
|
||||
b"User-agent: *\nDisallow: /acme/\n",
|
||||
# A rule for this app by name counts as well as the one for everyone.
|
||||
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
|
||||
.encode(),
|
||||
])
|
||||
def test_robots_that_disallow_the_page_fail_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
|
||||
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == []
|
||||
assert web["asked"] == [ROBOTS] # the listing is never fetched
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
||||
# The longest matching rule wins, whatever the order of the lines.
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
||||
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
|
||||
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
|
||||
# An allow wins a tie.
|
||||
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
|
||||
# `*` inside a path, and the query counts as part of it.
|
||||
("User-agent: *\nDisallow: /*company\n",
|
||||
"https://career5.successfactors.eu/career?company=x", False),
|
||||
# `$` anchors the end.
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
|
||||
# An empty Disallow allows.
|
||||
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
|
||||
# A byte order mark and CRLF line ends.
|
||||
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
|
||||
# Paths are case-sensitive.
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
|
||||
# A group for this app by name counts, a group for someone else does
|
||||
# not, and a token matches whole, not as a substring.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
|
||||
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
# Two agents share one group, and a comment is no rule.
|
||||
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
|
||||
"https://x.example/jobs/1", False),
|
||||
# A group for this app that allows does not undo the `*` group's refusal.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
# Rules before any user-agent line belong to no group.
|
||||
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
])
|
||||
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
|
||||
assert ats._robots_allows(robots.encode(), page) is allowed
|
||||
|
||||
|
||||
def test_a_robots_verdict_is_kept_for_a_day(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
module = _fake_host(robots=True)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
assert web["asked"] == [ROBOTS]
|
||||
|
||||
# Within the day the kept verdict holds, even though the site changed.
|
||||
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert (items, web["asked"]) == ([], [])
|
||||
|
||||
# A day on, robots.txt is read again.
|
||||
kept = json.loads(state["ats:robots:fake:acme"])
|
||||
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
||||
state["ats:robots:fake:acme"] = json.dumps(kept)
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
|
||||
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [404, 410, 401, 403])
|
||||
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
|
||||
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [
|
||||
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
|
||||
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
|
||||
])
|
||||
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == [] and web["asked"] == [ROBOTS]
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
|
||||
"robots.txt could not be read"
|
||||
assert "ats:robots:fake:acme" not in state
|
||||
|
||||
|
||||
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
|
||||
where a robots.txt that disallows everything is served."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if str(request.url) == ROBOTS:
|
||||
return httpx.Response(301, headers={"location": location})
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
|
||||
return httpx.Response(200, text='["1"]')
|
||||
return handler
|
||||
|
||||
|
||||
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", [
|
||||
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
|
||||
"http://www.jobs.fake.example/robots.txt",
|
||||
])
|
||||
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
||||
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Refused):
|
||||
ats._get(client, ROBOTS, "fake", "acme")
|
||||
|
||||
|
||||
def test_fetch_board_honours_robots_txt(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with pytest.raises(ats._Disallowed) as caught:
|
||||
ats.fetch_board("fake:acme")
|
||||
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- Lever EU
|
||||
|
||||
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||
assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom")
|
||||
assert not ats.is_known_target("lever:us/acme")
|
||||
assert not ats.is_known_target("lever:eu/")
|
||||
assert not ats.is_known_target("lever:eu/a/b")
|
||||
assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme"
|
||||
assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom"
|
||||
assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
|
||||
eu = ats._allowed_host("lever", "eu/tomtom")
|
||||
assert eu("api.eu.lever.co") and not eu("api.lever.co")
|
||||
assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co")
|
||||
|
||||
|
||||
def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
urls: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET"):
|
||||
urls.append(url)
|
||||
assert ats._allowed_host(host, slug)(httpx.URL(url).host)
|
||||
return (FIXTURES / "lever.json").read_bytes()
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type]
|
||||
assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"]
|
||||
assert rows and complete
|
||||
|
||||
|
||||
def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"})
|
||||
|
||||
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|
||||
+110
-10
@@ -4,7 +4,11 @@ site's pages are read."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import types
|
||||
from collections.abc import Set
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -36,16 +40,59 @@ def test_boards_in_names_readable_hosts_with_their_slug() -> None:
|
||||
|
||||
def test_boards_in_keeps_hosts_the_app_cannot_read() -> None:
|
||||
page = """
|
||||
<a href="https://novo.wd3.myworkdayjobs.com/en-US/careers">Workday</a>
|
||||
<a href="https://performancemanager.successfactors.eu/career?company=x">SF</a>
|
||||
<a href="https://novo.taleo.net/careersection/2/jobsearch.ftl">Taleo</a>
|
||||
<a href="https://acme.teamtailor.com/jobs">TT</a>
|
||||
"""
|
||||
assert boards_in(page) == [
|
||||
("workday", "novo", "https://novo.wd3.myworkdayjobs.com"),
|
||||
("successfactors", "", "performancemanager.successfactors.eu"),
|
||||
("taleo", "", ".taleo.net"),
|
||||
("teamtailor", "acme", "https://acme.teamtailor.com")]
|
||||
|
||||
|
||||
def test_boards_in_names_a_lever_eu_board_by_its_region() -> None:
|
||||
page = ('<a href="https://jobs.eu.lever.co/tomtom/abc-123">EU</a>'
|
||||
'<script src="https://api.eu.lever.co/v0/postings/tomtom?mode=json"></script>'
|
||||
'<a href="https://jobs.lever.co/whereby">US</a>')
|
||||
assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"),
|
||||
("lever", "whereby")]
|
||||
|
||||
|
||||
def test_boards_in_lowercases_a_slug_unless_the_host_keeps_its_case(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A Workday-shaped host with two groups and `KEEP_CASE`: the site keeps
|
||||
its case, and a site called `jobs` is a site, since only group 1 is
|
||||
checked against the hosts' own path words."""
|
||||
workday = re.compile(r"https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/"
|
||||
r"(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.IGNORECASE)
|
||||
monkeypatch.setattr(mod, "HOSTS", (("workday", workday),
|
||||
*(h for h in mod.HOSTS if h[0] != "workday")))
|
||||
monkeypatch.setattr(mod, "_KEEP_CASE", frozenset({"workday"}))
|
||||
page = ('<a href="https://lego.wd103.myworkdayjobs.com/LEGO_External">a</a>'
|
||||
'<a href="https://redhat.wd5.myworkdayjobs.com/en-US/jobs">b</a>'
|
||||
'<a href="https://jobs.smartrecruiters.com/BoschGroup">c</a>'
|
||||
'<a href="https://jobs.eu.lever.co/TomTom">d</a>')
|
||||
assert [(h, s) for h, s, _ in boards_in(page)] == [
|
||||
("workday", "lego.wd103/LEGO_External"), ("workday", "redhat.wd5/jobs"),
|
||||
("smartrecruiters", "boschgroup"), ("lever", "eu/tomtom")]
|
||||
# A host's own path word in group 1 still names no board.
|
||||
assert boards_in('<a href="https://jobs.smartrecruiters.com/api">x</a>') == []
|
||||
|
||||
|
||||
def test_no_module_of_the_eight_keeps_its_case() -> None:
|
||||
"""SmartRecruiters and Ashby ignore case (checked live, 2026-09-29), so a
|
||||
slug from a page stays lowercased as it always was."""
|
||||
assert not mod._KEEP_CASE
|
||||
|
||||
|
||||
def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
|
||||
"""Wave by wave, a host named here becomes readable by its module alone."""
|
||||
taleo: Any = types.ModuleType("taleo")
|
||||
taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",)
|
||||
hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE)
|
||||
assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [
|
||||
r"https?://([a-z0-9-]+)\.taleo\.net"]
|
||||
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
|
||||
|
||||
|
||||
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
|
||||
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
|
||||
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
|
||||
@@ -81,7 +128,7 @@ class Site:
|
||||
is a page that did not answer. Remembers the order of fetches."""
|
||||
|
||||
def __init__(self, pages: dict[str, tuple[str, str] | str],
|
||||
boards: set[tuple[str, str]] = frozenset()) -> None:
|
||||
boards: Set[tuple[str, str]] = frozenset()) -> None:
|
||||
self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()}
|
||||
self.boards = boards
|
||||
self.fetched: list[str] = []
|
||||
@@ -138,12 +185,12 @@ def test_a_careers_pages_own_link_is_followed_one_level_down() -> None:
|
||||
def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None:
|
||||
site = Site({
|
||||
"https://acme.com/": "<p>welcome</p>",
|
||||
"https://acme.com/careers": '<a href="https://acme.wd3.myworkdayjobs.com/x">Jobs</a>',
|
||||
"https://acme.com/careers": '<a href="https://acme.taleo.net/careersection/x">Jobs</a>',
|
||||
}, boards={("greenhouse", "acme")})
|
||||
found = run(site)
|
||||
assert found == Discovery(ats="workday", ats_slug="acme",
|
||||
assert found == Discovery(ats="taleo", ats_slug="",
|
||||
careers_url="https://acme.com/careers",
|
||||
evidence="https://acme.wd3.myworkdayjobs.com")
|
||||
evidence=".taleo.net")
|
||||
assert not found.readable
|
||||
# The page named the host: a probe could only find another tenant's
|
||||
# board of the same name, like the greenhouse:acme this fake web has.
|
||||
@@ -223,6 +270,59 @@ def test_slug_probes_find_a_board_the_pages_never_link() -> None:
|
||||
assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("boards", "found"), [
|
||||
({("greenhouse", "acme"), ("ashby", "acme")}, "greenhouse:acme"),
|
||||
({("lever", "acme"), ("ashby", "acme")}, "lever:acme"),
|
||||
({("smartrecruiters", "acme"), ("personio", "acme")}, "smartrecruiters:acme"),
|
||||
({("lever", "eu/acme"), ("personio", "acme")}, "lever:eu/acme"),
|
||||
])
|
||||
def test_a_slug_on_two_hosts_goes_to_the_one_probed_first(
|
||||
boards: set[tuple[str, str]], found: str) -> None:
|
||||
site = Site({}, boards=boards)
|
||||
assert run(site).board == found
|
||||
|
||||
|
||||
def test_the_probe_order_is_the_one_before_modules() -> None:
|
||||
assert list(mod._PROBES)[:7] == ["greenhouse", "lever", "ashby", "smartrecruiters",
|
||||
"recruitee", "personio", "teamtailor"]
|
||||
site = Site({})
|
||||
run(site)
|
||||
assert site.probed[:8] == [("greenhouse", "acme"), ("lever", "acme"), ("lever", "eu/acme"),
|
||||
("ashby", "acme"), ("smartrecruiters", "acme"),
|
||||
("recruitee", "acme"), ("personio", "acme"),
|
||||
("teamtailor", "acme")]
|
||||
|
||||
|
||||
def test_a_lever_eu_board_is_found_by_its_probe() -> None:
|
||||
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>"},
|
||||
boards={("lever", "eu/acme")})
|
||||
found = run(site)
|
||||
assert found.board == "lever:eu/acme"
|
||||
assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json"
|
||||
assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme"))
|
||||
|
||||
|
||||
def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None:
|
||||
asked: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
asked.append(str(request.url))
|
||||
if request.url.host == "api.eu.lever.co":
|
||||
return httpx.Response(200, json=[])
|
||||
if request.url.host == "api.ashbyhq.com":
|
||||
return httpx.Response(200, content=b'{"jobs":[]}')
|
||||
return httpx.Response(404)
|
||||
|
||||
with httpx.Client(transport=httpx.MockTransport(handler)) as client:
|
||||
assert mod.probe(client, "lever", "eu/acme")
|
||||
assert not mod.probe(client, "lever", "acme")
|
||||
assert not mod.probe(client, "ashby", "acme") # an empty listing is no board
|
||||
assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json",
|
||||
"https://api.lever.co/v0/postings/acme?limit=1&mode=json",
|
||||
"https://api.ashbyhq.com/posting-api/job-board/acme"]
|
||||
assert "workable" not in mod._PROBES and "lever" in mod._PROBES
|
||||
|
||||
|
||||
def test_a_site_that_never_answers_gives_an_empty_discovery() -> None:
|
||||
site = Site({})
|
||||
assert run(site) == Discovery()
|
||||
@@ -251,7 +351,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa
|
||||
{"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen",
|
||||
"board": "ashby:pleo"},
|
||||
{"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK",
|
||||
"city": "Bagsværd", "ats": "successfactors",
|
||||
"city": "Bagsværd", "ats": "taleo",
|
||||
"careers_url": "https://www.novonordisk.com/careers.html"},
|
||||
{"name": "Odd", "board": "nosuch:odd"},
|
||||
]}))
|
||||
@@ -266,7 +366,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa
|
||||
assert (sub.target, sub.company) == ("ashby:pleo", "Pleo")
|
||||
novo = store.company("novo-nordisk")
|
||||
assert novo is not None
|
||||
assert (novo.ats, novo.careers_url) == ("successfactors",
|
||||
assert (novo.ats, novo.careers_url) == ("taleo",
|
||||
"https://www.novonordisk.com/careers.html")
|
||||
odd = store.company("odd")
|
||||
assert odd is not None and odd.ats == ""
|
||||
|
||||
@@ -546,6 +546,24 @@ def test_replacing_or_detaching_a_board_forgets_that_it_was_failing(
|
||||
assert failing_boards(store) == {}
|
||||
|
||||
|
||||
def test_replacing_or_detaching_a_board_forgets_its_robots_and_detail_records(
|
||||
store: Store, boards: Boards) -> None:
|
||||
kept = ("ats:robots:", "ats:detailed:", "ats:detailfailed:")
|
||||
sources.set_board(store, "acme", "greenhouse:acme", name="Acme")
|
||||
for key in kept:
|
||||
store.state[key + "greenhouse:acme"] = "[]"
|
||||
store.state["ats:robots:lever:other"] = "{}"
|
||||
|
||||
sources.set_board(store, "acme", "ashby:acme")
|
||||
assert not any(key + "greenhouse:acme" in store.state for key in kept)
|
||||
assert "ats:robots:lever:other" in store.state
|
||||
|
||||
for key in kept:
|
||||
store.state[key + "ashby:acme"] = "[]"
|
||||
sources.detach_board(store, "acme")
|
||||
assert not any(key + "ashby:acme" in store.state for key in kept)
|
||||
|
||||
|
||||
def test_search_params_keeps_only_what_the_provider_reads() -> None:
|
||||
assert sources.search_params("jobindex", " python ", area=1, region="CaRE_1nn_cSU") == {
|
||||
"q": "python", "area": 1}
|
||||
|
||||
Reference in new issue
Block a user