Move the eight hosts into a module each under plugins/atshosts and build the ats plugin's tables from them

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 17:03:37 +02:00
1 parent 0a1d3c67c7
commit b02b72b4aa
12 files changed
+1296 -789

No files matched your search

+118 -784
View File
File diff suppressed because it is too large. Load diff
+113
View File
@@ -0,0 +1,113 @@
"""The hosts the `ats` plugin reads, one module each.
A host registers by existing: every module in this package whose name does
not start with `_` is loaded, in order of its name, and checked against the
contract below when the package is imported, so a module that breaks it fails
the import with a message naming the module and what it lacks, rather than a
pull that fails board by board. `plugins/ats.py` builds its allow-list,
feeds and pages from `HOSTS`, and `app/jobs/discover.py` its patterns and
probes. The package sits beside `ats.py` rather than being `plugins/ats/`,
since `desk.sources.registry` loads `plugins/*.py` by path and the tests
patch names on the `ats` module.
A module is named after its host and holds:
NAME = "lever" # the host of a target `lever:<slug>`, and the module's name
LABEL = "Lever" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: over a URL; the slug is group 1, or the
# non-empty groups joined by "/" when there are more
def slug_ok(slug) -> bool # strict; a slug that fails is no target
def allowed(slug) -> Callable[[str], bool] # the host names a fetch may reach
def careers_url(slug) -> str # the page a person opens
and one of two ways to read a board:
def feed_url(slug) -> str # the whole board in one answer,
def parse(body: bytes, slug) -> list[Row] # read with `parse`
def page(get: Get, slug, cursor) -> Page # a paged board, one page a call,
def rows(decoded, slug) -> list[Row] # its rows from `page`'s decoded page
and, optionally:
def fallback_url(slug) -> str # a smaller feed, read once when
# `feed_url`'s answer is over the byte cap
def robots_page(slug) -> str # the URL the site's robots.txt must allow
def detail(get: Get, slug, row: Row) -> Row # the row filled from the role's own page
def probe_url(slug) -> str # discovery: answers 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess) -> tuple[str, ...] # the slugs a guessed name is probed as
`Get`, `Row` and `Page` are in `_shared`, with the helpers several hosts use.
A host module imports nothing from `plugins.ats`: the fetch it is handed is
`ats._get` bound to the board, so every request a host makes passes the same
allow-list, redirect rule, byte cap and deadline.
"""
from __future__ import annotations
import importlib
import pkgutil
import re
from collections.abc import Iterable
from types import ModuleType
#: What every host module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal."""
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
out.append("LABEL is not a non-empty string")
patterns = getattr(module, "PATTERNS", ())
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
out.append("PATTERNS is not a tuple of strings")
else:
for pattern in patterns:
try:
re.compile(pattern)
except (TypeError, re.error) as exc:
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
out += [f"{attr} is not callable" for attr in FUNCTIONS
if hasattr(module, attr) and not callable(getattr(module, attr))]
if hasattr(module, "page"):
if not hasattr(module, "rows"):
out.append("page without rows")
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
out.append("neither feed_url and parse nor page and rows")
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
out.append("probe_ok without probe_url")
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
out.append("probe_slugs without probe_url")
return out
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
"""`NAME -> module` for every module of `package` (found on `path`) whose
name does not start with `_`, sorted by name. Raises ImportError naming
the first module that breaks the contract."""
hosts: dict[str, ModuleType] = {}
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{package}.{info.name}")
if found := problems(module, info.name):
raise ImportError(f"{package}.{info.name} breaks the host contract: "
+ "; ".join(found))
hosts[module.NAME] = module
return hosts
#: host -> its module, in order of name.
HOSTS: dict[str, ModuleType] = load(__path__, __name__)
+188
View File
@@ -0,0 +1,188 @@
"""What the host modules share: the row and page shapes, the fetch they are
handed, and the date, URL and salary helpers more than one host needs.
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
import it without either importing the other. The leading underscore keeps
`plugins.atshosts` from listing it as a host.
"""
from __future__ import annotations
import re
from collections.abc import Callable
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from typing import Any, Protocol
import httpx
#: What every parser turns a feed into: one dict per row, with the keys
#: `source_id`, `title`, `location`, `url`, `posted`, `department`,
#: `description`, `remote` and `salary_text`, and, where a host states pay,
#: `salary_min`, `salary_max` and `currency`.
Row = dict[str, Any]
#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded
#: page, its raw results (dicts with an `id`, counted for completeness), the
#: total it states (None for a host that states none), and the cursor of the
#: next page (None after the last). The first call gets cursor None.
Page = tuple[Any, list[dict[str, Any]], Any, Any]
class Get(Protocol):
"""The fetch a host is handed: `ats.py`'s own, bound to one board, so the
allow-list, the redirect rule, the byte cap and the deadline apply to
every request a host makes. A POST sends `payload` as JSON; `headers` are
added to (and may replace) the default User-Agent and Accept. Returns the
body, and raises on a non-2xx answer as `httpx` does."""
def __call__(self, url: str, *, method: str = "GET",
payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes: ...
def is_web_url(url: str) -> bool:
"""A trimmed http(s) URL, or refuse — a feed is external content and a
`javascript:` or `data:` link must never reach an href. Port of
openings.go's `webURL`."""
url = (url or "").strip()
if not url:
return False
parsed = httpx.URL(url)
return parsed.scheme in ("http", "https") and bool(parsed.host)
def only(host: str) -> Callable[[str], bool]:
"""The allow-list of a board that is fetched from one host name only,
which is every host so far."""
return lambda h: h == host
#: A slug that is one URL path segment: what the hosts that put the slug in
#: the path accept, and nothing that could add a path, a query or a fragment.
_SEGMENT = re.compile(r"^[A-Za-z0-9._~%-]+$")
#: A slug that is one DNS label, for the hosts that put it in the host name.
_LABEL = re.compile(r"^[A-Za-z0-9-]+$")
def path_slug_ok(slug: str) -> bool:
return bool(_SEGMENT.match(slug)) and slug not in (".", "..")
def label_slug_ok(slug: str) -> bool:
return bool(_LABEL.match(slug))
def parse_dt(value: Any) -> datetime | None:
"""An aware datetime from whatever a board sends: ISO-8601 with an
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
if value is None:
return None
if isinstance(value, int | float):
if value <= 0:
return None
seconds = value / 1000 if value > 10**12 else value
return datetime.fromtimestamp(seconds, tz=UTC)
text = str(value).strip()
if not text:
return None
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
if not match:
try:
parsed = parsedate_to_datetime(text)
except (TypeError, ValueError):
return None
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
# ---------------------------------------------------------- structured pay
#
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
# `raw` into, per docs/decisions.md's "raw is transport, columns are
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
# structured pay on their list endpoint; Workable, SmartRecruiters and
# Teamtailor do not (checked live, 2026-09-29), so their rows leave the
# columns out and `ATSSource.fetch` reads a plain '' or None for them.
def num(value: Any) -> float | None:
"""A positive number from whatever a board sends (int, float, or a
numeral string), or None for anything else, including zero and a
negative — a board's placeholder 0 (an unfilled field, not a real
figure) must not become a salary floor of zero."""
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if number > 0 else None
def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
None, "")` when there is no currency, no bound, or `interval` is not one
of `factors`' own keys — which is how an hourly or a daily rate is left
out rather than annualised into a number nobody stated. `factors` maps
the board's own interval spelling to the multiplier that makes it
yearly: 1 for a figure already yearly, 12 for a monthly one."""
code = str(currency or "").strip().upper()
factor = factors.get(str(interval or "").strip())
if not code or not factor:
return None, None, ""
lo, hi = num(min_value), num(max_value)
if lo is None and hi is None:
return None, None, ""
return (int(lo * factor) if lo is not None else None,
int(hi * factor) if hi is not None else None, code)
# ------------------------------------------------------------- salary_text
#
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
# hourly and daily rates, a Greenhouse role with several pay zones, and a
# range with no currency are all dropped from those columns rather than
# guessed into a wrong yearly figure. `salary_text` is where that dropped
# information survives — one human-readable line per parser, written
# whether or not the pay conformed, so a person can read it and do their own
# sums and a stricter analysis can find the roles the columns left out
# (`Role.salary_state`, `app/models.py`).
#: Formats a number the way a salary line writes it: no decimals for a whole
#: figure (the normal case for a yearly or monthly one), two for anything
#: else (an hourly or daily rate quoted with cents).
def fmt_amount(value: float) -> str:
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
def bounds_words(lo: float | None, hi: float | None) -> str:
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
bound stated, or "" with neither."""
if lo is not None and hi is not None:
return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}"
if lo is not None:
return f"from {fmt_amount(lo)}"
if hi is not None:
return f"up to {fmt_amount(hi)}"
return ""
def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
per year" — or "" when there is no bound to report at all. A currency
that was not stated does not drop the line, unlike the normalised
columns: it is said plainly instead, "80,000 to 100,000 per year (no
currency stated)", so the figure is not lost along with the currency."""
bounds = bounds_words(lo, hi)
if not bounds:
return ""
tail = f" per {period}" if period else ""
code = str(currency or "").strip().upper()
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
+121
View File
@@ -0,0 +1,121 @@
"""Ashby: `ashby:<job board name>`, the whole board in one JSON answer. A
board name may carry a dot (`jimdo.com`) or be a uuid.
The field map is ported from `openings.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
NAME = "ashby"
LABEL = "Ashby"
PATTERNS = (
r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)",
r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)",
)
_API = "api.ashbyhq.com"
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_API)
def careers_url(slug: str) -> str:
return f"https://jobs.ashbyhq.com/{slug}"
def feed_url(slug: str) -> str:
return f"https://{_API}/posting-api/job-board/{slug}?includeCompensation=true"
def probe_url(slug: str) -> str:
return f"https://{_API}/posting-api/job-board/{slug}"
def probe_ok(body: bytes) -> bool:
"""Ashby answers an empty listing for an unknown name, so a listing
without jobs counts as no board."""
return b'"jobs":[' in body and b'"jobs":[]' not in body
def _salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
"""The yearly range from `compensation.summaryComponents` (needs
`?includeCompensation=true` on the feed URL, see `feed_url`): the first
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
an hourly one is left out rather than annualised."""
for component in components:
if component.get("compensationType") != "Salary":
continue
return yearly_amount(component.get("minValue"), component.get("maxValue"),
component.get("currencyCode"), component.get("interval"),
{"1 YEAR": 1, "1 MONTH": 12})
return None, None, ""
def _salary_text(compensation: dict[str, Any]) -> str:
"""One line per `summaryComponents` entry that states a bound, labelled
by the board's own `compensationType` when more than one does (a salary
range beside a bonus range, say); with just the one, the label is left
off, the same as Greenhouse's. `interval` is "<n> <PERIOD>" ("1 YEAR",
"1 HOUR"); the last word, lowered, is the period. With nothing to build
a line from, Ashby's own prose summary (`compensationTierSummary`, else
`scrapeableCompensationSalarySummary`) is what is left to show."""
components = compensation.get("summaryComponents") or []
stated = [c for c in components if num(c.get("minValue")) is not None
or num(c.get("maxValue")) is not None]
lines = []
for component in stated:
period = str(component.get("interval") or "").split(" ")[-1].lower()
line = pay_line(num(component.get("minValue")), num(component.get("maxValue")),
component.get("currencyCode"), period)
if not line:
continue
if len(stated) > 1:
label = str(component.get("compensationType") or "").strip()
line = f"{label}: {line}" if label else line
lines.append(line)
if lines:
return "; ".join(lines)
return str(compensation.get("compensationTierSummary")
or compensation.get("scrapeableCompensationSalarySummary") or "")
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
for j in data.get("jobs") or []:
title = (j.get("title") or "").strip()
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
if not title or not is_web_url(url):
continue
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
compensation = j.get("compensation") or {}
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": (j.get("location") or "").strip(),
"url": url,
"posted": parse_dt(j.get("publishedAt")),
"department": (j.get("department") or j.get("team") or "").strip(),
"description": strip_html(description),
"remote": j.get("isRemote"),
"salary_text": _salary_text(compensation),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+138
View File
@@ -0,0 +1,138 @@
"""Greenhouse: `greenhouse:<board token>`, the whole board in one JSON answer.
The field map is ported from `openings.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
"""
from __future__ import annotations
import html
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line
NAME = "greenhouse"
LABEL = "Greenhouse"
PATTERNS = (
r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)",
r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)",
)
_API = "boards-api.greenhouse.io"
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_API)
def careers_url(slug: str) -> str:
return f"https://job-boards.greenhouse.io/{slug}"
def feed_url(slug: str) -> str:
return f"https://{_API}/v1/boards/{slug}/jobs?content=true&pay_transparency=true"
def fallback_url(slug: str) -> str:
"""The feed without `content=true`, read once when the full one exceeds
the byte cap: the descriptions are lost, but the titles land rather than
the whole board being skipped."""
return f"https://{_API}/v1/boards/{slug}/jobs"
def probe_url(slug: str) -> str:
return f"https://{_API}/v1/boards/{slug}"
def _salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
on the feed URL, see `feed_url`), which the list endpoint already carries
in the same request — no per-role fetch. Only when there is exactly one
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
one per office) gives no way to tell which office the role in hand is
at, and picking one would misrepresent the columns. Greenhouse's own
ranges are cents, and yearly by convention except an hourly or a daily
one, named as such in `title` with no separate field to check — those
are skipped rather than annualised into a number nobody stated."""
if len(ranges) != 1:
return None, None, ""
entry = ranges[0]
title = str(entry.get("title") or "").lower()
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
return None, None, ""
code = str(entry.get("currency_type") or "").strip().upper()
if not code:
return None, None, ""
lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents"))
if lo is None and hi is None:
return None, None, ""
return (int(lo / 100) if lo is not None else None,
int(hi / 100) if hi is not None else None, code)
def _salary_text(ranges: list[dict[str, Any]]) -> str:
"""One line per `pay_input_ranges` entry, joined with "; " — every range
Greenhouse states, not only the single one `_salary` accepts for the
columns. With more than one, each is labelled with the board's own
`title` ("Zone 1 Pay Range: ..."), since that is the only way to tell the
ranges apart; with just the one, the label adds nothing and is left off.
The hour/day check is the same one `_salary` uses to keep those out of
the yearly columns; here it only picks the word that follows "per"."""
multi = len(ranges) > 1
lines = []
for entry in ranges:
title = str(entry.get("title") or "").strip()
low_title = title.lower()
period = ("hour" if any(w in low_title for w in ("hour", "/hr"))
else "day" if "day" in low_title else "year")
lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents"))
line = pay_line(lo / 100 if lo is not None else None,
hi / 100 if hi is not None else None,
entry.get("currency_type"), period)
if not line:
continue
if multi:
label = title.rstrip(":").strip() or "Range"
line = f"{label}: {line}"
lines.append(line)
return "; ".join(lines)
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
for j in data.get("jobs") or []:
title = (j.get("title") or "").strip()
url = (j.get("absolute_url") or "").strip()
if not title or not is_web_url(url):
continue
departments = j.get("departments") or []
# Greenhouse ships the description HTML-entity-escaped inside the
# JSON string (`&lt;p&gt;`), so it needs unescaping before the tags
# themselves can be stripped.
content = html.unescape(j.get("content") or "")
ranges = j.get("pay_input_ranges") or []
salary_min, salary_max, currency = _salary(ranges)
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": ((j.get("location") or {}).get("name") or "").strip(),
"url": url,
"posted": parse_dt(j.get("first_published") or j.get("updated_at")),
"department": (departments[0].get("name") if departments else "") or "",
"description": strip_html(content),
"remote": None, # Greenhouse's feed does not say
"salary_text": _salary_text(ranges),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+120
View File
@@ -0,0 +1,120 @@
"""Lever: `lever:<site>`, or `lever:eu/<site>` for a board on Lever's EU
instance, the whole board in one JSON answer.
Lever runs two instances that share nothing but the API: `api.lever.co` and
`api.eu.lever.co`, with the pages on `jobs.lever.co` and `jobs.eu.lever.co`.
A site exists on one of them only (TomTom and Prima answer 404 on the other,
2026-09-29), and the same name on both would be two companies, so the region
is part of the slug. It is a prefix a plain site name cannot carry, since a
Lever site name has no slash, which leaves every `lever:<site>` target as it
was.
The field map is ported from `openings.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
NAME = "lever"
LABEL = "Lever"
#: Two groups on the EU hosts, `eu` and the site, which discovery joins into
#: the slug `eu/<site>`.
PATTERNS = (
r"jobs\.lever\.co/([A-Za-z0-9_-]+)",
r"api\.lever\.co/v0/postings/([A-Za-z0-9_-]+)",
r"jobs\.(eu)\.lever\.co/([A-Za-z0-9_-]+)",
r"api\.(eu)\.lever\.co/v0/postings/([A-Za-z0-9_-]+)",
)
#: The region prefixes a slug may carry, to the infix of both host names.
_REGIONS = {"eu": "eu."}
def _split(slug: str) -> tuple[str, str]:
"""`(host infix, site)`: `("eu.", "tomtom")` for `eu/tomtom`, `("", "acme")`
for `acme`."""
region, sep, site = slug.partition("/")
return (_REGIONS.get(region, "?"), site) if sep else ("", slug)
def slug_ok(slug: str) -> bool:
infix, site = _split(slug)
return infix != "?" and path_slug_ok(site)
def allowed(slug: str) -> Callable[[str], bool]:
return only(f"api.{_split(slug)[0]}lever.co")
def careers_url(slug: str) -> str:
infix, site = _split(slug)
return f"https://jobs.{infix}lever.co/{site}"
def feed_url(slug: str) -> str:
infix, site = _split(slug)
return f"https://api.{infix}lever.co/v0/postings/{site}?mode=json"
def probe_url(slug: str) -> str:
infix, site = _split(slug)
return f"https://api.{infix}lever.co/v0/postings/{site}?limit=1&mode=json"
def probe_slugs(guess: str) -> tuple[str, ...]:
"""A name is tried on both instances, the US one first, since a site that
names neither leaves no other way to tell which one it is on."""
return (guess, *(f"{region}/{guess}" for region in _REGIONS))
def _salary_text(salary_range: dict[str, Any]) -> str:
"""One line from `salaryRange`, whatever its `interval` — including an
hourly or daily one, which `yearly_amount` drops from the columns.
Lever's interval is "per-<period>-<kind>" ("per-year-salary",
"per-hour-wage"); the middle word is the period."""
interval = str(salary_range.get("interval") or "")
parts = interval.split("-")
period = parts[1] if len(parts) >= 3 else ""
return pay_line(num(salary_range.get("min")), num(salary_range.get("max")),
salary_range.get("currency"), period)
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
if not isinstance(data, list):
return []
rows = []
for j in data:
title = (j.get("text") or "").strip()
url = (j.get("hostedUrl") or "").strip()
if not title or not is_web_url(url):
continue
categories = j.get("categories") or {}
description = j.get("description") or j.get("descriptionPlain") or ""
salary_range = j.get("salaryRange") or {}
salary_min, salary_max, currency = yearly_amount(
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": (categories.get("location") or "").strip(),
"url": url,
"posted": parse_dt(j.get("createdAt")),
"department": (categories.get("department") or categories.get("team") or "").strip(),
"description": strip_html(description),
"remote": (j.get("workplaceType") or "").strip() or None,
"salary_text": _salary_text(salary_range),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+100
View File
@@ -0,0 +1,100 @@
"""Personio: `personio:<subdomain>`, the whole board in one XML answer from
the company's own subdomain of jobs.personio.de.
Written from one real board on 2026-09-23; jobhunt-hq did not read
Personio.
"""
from __future__ import annotations
import xml.etree.ElementTree as ET
from collections.abc import Callable
from desk.sources import strip_html
from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "personio"
LABEL = "Personio"
PATTERNS = (r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)",)
#: Personio's `salaryInformation/type` spellings, to the plain period word
#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through.
_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
"daily": "day", "weekly": "week"}
def slug_ok(slug: str) -> bool:
return label_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
"""The company's own subdomain, so one company's slug cannot pass for
another's."""
return only(f"{slug}.jobs.personio.de")
def careers_url(slug: str) -> str:
return f"https://{slug}.jobs.personio.de"
def feed_url(slug: str) -> str:
return f"https://{slug}.jobs.personio.de/xml"
def probe_url(slug: str) -> str:
return feed_url(slug)
def probe_ok(body: bytes) -> bool:
return b"<position>" in body
def _salary_text(salary_info: ET.Element) -> str:
"""One line from a `<salaryInformation>` element, whatever its `type`,
including an hourly or daily one that `yearly_amount` drops from the
columns."""
period_raw = (salary_info.findtext("type") or "").strip().lower()
period = _PERIODS.get(period_raw, period_raw)
return pay_line(num(salary_info.findtext("min")), num(salary_info.findtext("max")),
salary_info.findtext("currencyCode"), period)
def parse(body: bytes, slug: str) -> list[Row]:
"""`{slug}.jobs.personio.de/xml` -> `<workzag-jobs><position>...`. No job
URL in the feed itself; Personio's own page is `/job/{id}`, confirmed
live against a real posting (2026-09-23)."""
root = ET.fromstring(body)
rows = []
for position in root.findall("position"):
job_id = (position.findtext("id") or "").strip()
title = (position.findtext("name") or "").strip()
if not job_id or not title:
continue
description = " ".join(
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
salary_info = position.find("salaryInformation")
salary_min = salary_max = None
currency = ""
salary_text = ""
if salary_info is not None:
salary_min, salary_max, currency = yearly_amount(
salary_info.findtext("min"), salary_info.findtext("max"),
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
{"yearly": 1, "monthly": 12})
salary_text = _salary_text(salary_info)
rows.append({
"source_id": job_id,
"title": title,
"location": (position.findtext("office") or "").strip(),
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
"posted": parse_dt(position.findtext("createdAt")),
"department": (position.findtext("department") or "").strip(),
"description": strip_html(description),
"remote": "remote" in (position.findtext("office") or "").lower() or None,
"salary_text": salary_text,
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+83
View File
@@ -0,0 +1,83 @@
"""Recruitee: `recruitee:<subdomain>`, the whole board in one JSON answer
from the company's own subdomain of recruitee.com.
Written from one real board on 2026-09-23; jobhunt-hq did not read
Recruitee.
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "recruitee"
LABEL = "Recruitee"
PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",)
def slug_ok(slug: str) -> bool:
return label_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
"""The company's own subdomain, so one company's slug cannot pass for
another's."""
return only(f"{slug}.recruitee.com")
def careers_url(slug: str) -> str:
return f"https://{slug}.recruitee.com"
def feed_url(slug: str) -> str:
return f"https://{slug}.recruitee.com/api/offers/"
def probe_url(slug: str) -> str:
return feed_url(slug)
def probe_ok(body: bytes) -> bool:
return b'"offers":[' in body and b'"offers":[]' not in body
def _salary_text(salary: dict[str, Any]) -> str:
"""One line from Recruitee's `salary` object; `period` is already a
plain word ("year", "month")."""
period = str(salary.get("period") or "").strip().lower()
return pay_line(num(salary.get("min")), num(salary.get("max")),
salary.get("currency"), period)
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
for j in data.get("offers") or []:
title = (j.get("title") or "").strip()
url = (j.get("careers_url") or "").strip()
if not title or not is_web_url(url):
continue
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
salary_min, salary_max, currency = yearly_amount(
salary.get("min"), salary.get("max"), salary.get("currency"),
salary.get("period"), {"year": 1, "month": 12})
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": (j.get("location") or j.get("city") or "").strip(),
"url": url,
"posted": parse_dt(j.get("published_at") or j.get("created_at")),
"department": (j.get("department") or "").strip(),
"description": strip_html(j.get("description") or ""),
"remote": j.get("remote"),
"salary_text": _salary_text(salary),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
+89
View File
@@ -0,0 +1,89 @@
"""SmartRecruiters: `smartrecruiters:<company identifier>`, paged by offset
with a stated total. The identifier is case-sensitive (`BoschGroup`).
Written from one real board on 2026-09-23; jobhunt-hq did not read
SmartRecruiters.
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
NAME = "smartrecruiters"
LABEL = "SmartRecruiters"
PATTERNS = (
r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)",
)
_API = "api.smartrecruiters.com"
#: The page size asked for, which is also the most the host serves per page.
PAGE_SIZE = 100
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_API)
def careers_url(slug: str) -> str:
return f"https://jobs.smartrecruiters.com/{slug}"
def probe_url(slug: str) -> str:
return f"https://{_API}/v1/companies/{slug}/postings?limit=1"
def probe_ok(body: bytes) -> bool:
"""An unknown identifier answers an empty listing, so a listing without
postings counts as no board."""
return b'"totalFound":' in body and b'"totalFound":0' not in body
def page(get: Get, slug: str, cursor: Any) -> Page:
offset = int(cursor or 0)
data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings"
f"?offset={offset}&limit={PAGE_SIZE}"))
content = data.get("content") or []
return data, content, data.get("totalFound"), offset + len(content) if content else None
def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
def rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
page, whatever `limit` asks for; the next is `?offset=` further on. The
list omits the job description (only the per-posting detail endpoint has
it, and fetching one per row would be hundreds of extra requests per
board), so `description` stays empty here."""
out = []
for j in data.get("content") or []:
title = (j.get("name") or "").strip()
job_id = str(j.get("id") or "")
if not title or not job_id:
continue
location = j.get("location") or {}
loc_text = (location.get("fullLocation")
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
out.append({
"source_id": job_id,
"title": title,
"location": (loc_text or "").strip(),
"url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}",
"posted": parse_dt(j.get("releasedDate")),
"department": ((j.get("department") or {}).get("label") or "").strip(),
"description": "",
"remote": location.get("remote"),
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
})
return out
+136
View File
@@ -0,0 +1,136 @@
"""Teamtailor: `teamtailor:<subdomain>` for acme.teamtailor.com, or, for a
career site on the company's own domain, that whole host name:
`teamtailor:careers.acme.com`. An RSS feed paged by offset that states no
total.
Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor.
"""
from __future__ import annotations
import html
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "teamtailor"
LABEL = "Teamtailor"
PATTERNS = (
r"https?://([a-z0-9-]+)\.teamtailor\.com",
# No slug: a career site on the company's own domain, which only loads
# Teamtailor's CDN. Discovery reads its board off the page's host.
r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets",
)
#: The page size, asked for as `per_page`. A feed with neither parameter
#: answers with every role (73 of 73 at Shine, 2026-09-29), but says nowhere
#: that it is the whole list, so this reads by page to an empty one;
#: `per_page=200` was answered in full too, and the cursor advances by what a
#: page returned, so a smaller cap on the host's side would not skip a role.
PAGE_SIZE = 50
#: Only a DNS name with a letter in its last label is a site, so an address
#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could
#: be pointed at; `ats._get` still refuses every host but this one, redirects
#: included.
_HOST = re.compile(r"^[a-z0-9-]+(\.[a-z0-9-]+)*$")
#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`.
_TT = "{https://teamtailor.com/locations}"
#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing
#: about where the role is worked and `temporary` is remote for a spell only,
#: so both stay unknown rather than become "onsite" or "remote".
_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"}
def site(slug: str) -> str:
"""The host a career site is served from: `acme` is acme.teamtailor.com,
and a slug with a dot is a site on the company's own domain,
`careers.acme.com`, taken whole (the feed path is the same on both)."""
slug = slug.strip().lower()
return slug if "." in slug else f"{slug}.teamtailor.com"
def slug_ok(slug: str) -> bool:
host = site(slug)
return bool(slug.strip()) and bool(_HOST.match(host)) \
and not host.rsplit(".", 1)[-1].isdigit()
def allowed(slug: str) -> Callable[[str], bool]:
return only(site(slug))
def careers_url(slug: str) -> str:
return f"https://{site(slug)}/jobs"
def probe_url(slug: str) -> str:
return f"https://{site(slug)}/jobs.rss"
def probe_ok(body: bytes) -> bool:
return b"<item>" in body
def page(get: Get, slug: str, cursor: Any) -> Page:
offset = int(cursor or 0)
items = _items(get(f"https://{site(slug)}/jobs.rss?per_page={PAGE_SIZE}&offset={offset}"))
results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items]
return items, results, None, offset + len(items) if items else None
def parse(body: bytes, slug: str) -> list[Row]:
return rows(_items(body), slug)
def _items(body: bytes) -> list[ET.Element]:
"""`{site}/jobs.rss` -> `<rss><channel><item>...`, the feed every Teamtailor
career site serves (checked live on a dozen sites, 2026-09-29). A missing
`channel` is a page that is not a feed, and raises."""
channel = ET.fromstring(body).find("channel")
if channel is None:
raise ValueError("not a Teamtailor feed: no channel")
return channel.findall("item")
def rows(items: list[ET.Element], slug: str) -> list[Row]:
"""One page of items. The item has the title, the job URL (on the site's
own domain), a `guid` uuid, `pubDate`, the HTML `description` in full,
`remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the
`tt:` namespace, `locations` (a list: `city` and `country` each, `name` free
text), `department` and `role`, either of which may be empty. `place` joins
the distinct locations with "; ", which `country_of` reads a location at a
time. No pay is stated anywhere in the feed."""
out = []
for item in items:
title = (item.findtext("title") or "").strip()
url = (item.findtext("link") or "").strip()
if not title or not is_web_url(url):
continue
places: list[str] = []
for loc in item.findall(f"{_TT}locations/{_TT}location"):
city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip()
country = (loc.findtext(f"{_TT}country") or "").strip()
place = city if country == city else ", ".join(p for p in (city, country) if p)
if place and place not in places:
places.append(place)
out.append({
"source_id": (item.findtext("guid") or url).strip(),
"title": title,
"location": "; ".join(places),
"url": url,
"posted": parse_dt(item.findtext("pubDate")),
"department": ((item.findtext(f"{_TT}department") or "").strip()
or (item.findtext(f"{_TT}role") or "").strip()),
"description": html.unescape(strip_html(item.findtext("description") or "")),
"remote": _REMOTE.get((item.findtext("remoteStatus") or "").strip()),
"salary_text": "", # Teamtailor's feed states no pay at all
})
return out
+84
View File
@@ -0,0 +1,84 @@
"""Workable: `workable:<account>`, paged ten roles at a time with a stated
total.
Written from one real board, Hugging Face's, on 2026-09-28, when the first
board with roles was seen; jobhunt-hq did not read Workable.
"""
from __future__ import annotations
import json
from collections.abc import Callable
from typing import Any
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
NAME = "workable"
LABEL = "Workable"
PATTERNS = (
r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)",
r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com",
)
_API = "apply.workable.com"
# No probe: Workable's API sits behind a rate limit that a run of guesses
# trips, and a Workable site links its board plainly anyway.
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_API)
def careers_url(slug: str) -> str:
return f"https://{_API}/{slug}"
def page(get: Get, slug: str, cursor: Any) -> Page:
"""One POST of the list; the next page is the same POST with
`{"token": nextPage}`, and the last page has no `nextPage`."""
url = f"https://{_API}/api/v3/accounts/{slug}/jobs"
data = json.loads(get(url, method="POST", payload={"token": cursor} if cursor else {}))
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
def rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
Face's board answered on 2026-09-28. A result carries no URL and no text:
the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and
the description is only on the per-job endpoint, one request per role,
so `description` stays empty here as it does for SmartRecruiters.
`location` is an object and `department` a list; `remote` is a flag
beside `workplace` (remote, hybrid, on_site)."""
out = []
for j in data.get("results") or []:
title = (j.get("title") or "").strip()
shortcode = (j.get("shortcode") or "").strip()
if not title or not shortcode:
continue
loc = j.get("location") or {}
location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p)
dept = j.get("department") or ""
if isinstance(dept, list):
dept = ", ".join(d for d in dept if d)
out.append({
"source_id": str(j.get("id") or shortcode),
"title": title,
"location": location,
"url": f"https://{_API}/{slug}/j/{shortcode}/",
"posted": parse_dt(j.get("published")),
"department": dept.strip(),
"description": "",
"remote": bool(j.get("remote")) or j.get("workplace") == "remote",
"salary_text": "", # Workable's list endpoint states no pay at all
})
return out
+6 -5
View File
@@ -19,6 +19,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats # noqa: E402
from plugins.atshosts import smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -384,7 +385,7 @@ def test_is_known_target_accepts_the_eight_hosts_and_refuses_others() -> None:
"recruitee", "personio", "teamtailor"):
assert ats.is_known_target(f"{host}:acme")
assert not ats.is_known_target("linkedin:acme")
assert not ats.is_known_target("workday:acme")
assert not ats.is_known_target("taleo:acme")
assert not ats.is_known_target("greenhouse:")
assert not ats.is_known_target("nohostnoseparator")
# a Teamtailor slug names a host: its own subdomain, or a whole domain
@@ -420,7 +421,7 @@ def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
assert ats.careers_url("teamtailor:acme") == "https://acme.teamtailor.com/jobs"
assert ats.careers_url("teamtailor:careers.acme.com") == "https://careers.acme.com/jobs"
assert ats.careers_url("workday:acme") == ""
assert ats.careers_url("taleo:acme") == ""
# --------------------------------------------------------------- body cap
@@ -570,7 +571,7 @@ def test_smartrecruiters_pages_by_offset_to_total_found(
urls.append(url)
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
@@ -588,7 +589,7 @@ def test_a_board_that_ends_short_of_its_total_is_incomplete(
def fake_get(client, url, host, slug, *, method="GET", payload=None):
return empty if "offset=2" in url else _page("smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
@@ -604,7 +605,7 @@ def test_a_repeated_row_does_not_count_twice_toward_the_total(
page["content"] = page["content"][1:]
return json.dumps(page).encode()
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)