Read Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee and
Personio boards The `ats` plugin: one Source per host's public JSON (Personio: XML), each parsed to a common row and yielded as a `role` item. The field map, the redirect allow-list, the 8 MB cap and the 10 s timeout are ported from jobhunt-hq's openings.go (MIT); the nine-function classifier is ported from its seedgen. Workable, SmartRecruiters, Recruitee and Personio are new, written from one real board of each fetched while this was built. Fixtures under tests/fixtures/ats/ keep every test off the network. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ac5eca874a
commit
21ed472a87
10 files changed
+909
No files matched your search
@@ -0,0 +1,8 @@
|
||||
"""Drop-in plugins, discovered by `plugin_paths` in `roles.yaml`.
|
||||
|
||||
A plugin file imports `desk` only — see `tests/test_architecture.py`. This
|
||||
`__init__.py` makes the directory importable as `plugins.<name>` for the rest
|
||||
of roledesk (`jobs/pull.py`, `cli.py`, the tests), which the plugin contract
|
||||
itself never relies on: `desk.sources.registry` finds `ats.py` by path, not
|
||||
by this package.
|
||||
"""
|
||||
+548
@@ -0,0 +1,548 @@
|
||||
"""ATS boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio.
|
||||
|
||||
Every one of these hosts is a company's own careers page speaking public,
|
||||
unauthenticated JSON (Personio: XML) — first-party, no scraping, and a role
|
||||
that drops off the board is *closed*, which is the cheapest staleness check
|
||||
there is.
|
||||
|
||||
The field map, the redirect allow-list, the 8 MB body cap and the 10 s
|
||||
timeout are ported from `openings.go` and `cmd/seedgen/main.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
||||
`classify`'s nine functions and their keywords. Workable, SmartRecruiters,
|
||||
Recruitee and Personio are new: jobhunt-hq only read Greenhouse, Lever and
|
||||
Ashby, so those four are written from one real board of each host, fetched
|
||||
while this file was written (2026-09-23), rather than from the Go source.
|
||||
|
||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||
`lever:acme`. Every board is fetched independently; one that fails or times
|
||||
out is logged and skipped, and the rest continue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import (
|
||||
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: Matches openings.go: a board answers in 10s or it is skipped, and its body
|
||||
#: is capped at 8 MB so a misbehaving host cannot hang a pull or fill memory.
|
||||
TIMEOUT = 10.0
|
||||
MAX_BYTES = 8 << 20
|
||||
|
||||
#: host -> (feed URL template, careers URL template). `{slug}` is filled in;
|
||||
#: recruitee and personio put the slug in the hostname instead of the path.
|
||||
_BOARDS: dict[str, tuple[str, str]] = {
|
||||
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs?content=true",
|
||||
"https://job-boards.greenhouse.io/{slug}"),
|
||||
"lever": ("https://api.lever.co/v0/postings/{slug}?mode=json",
|
||||
"https://jobs.lever.co/{slug}"),
|
||||
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}",
|
||||
"https://jobs.ashbyhq.com/{slug}"),
|
||||
"workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs",
|
||||
"https://apply.workable.com/{slug}"),
|
||||
"smartrecruiters": ("https://api.smartrecruiters.com/v1/companies/{slug}/postings",
|
||||
"https://jobs.smartrecruiters.com/{slug}"),
|
||||
"recruitee": ("https://{slug}.recruitee.com/api/offers/",
|
||||
"https://{slug}.recruitee.com"),
|
||||
"personio": ("https://{slug}.jobs.personio.de/xml",
|
||||
"https://{slug}.jobs.personio.de"),
|
||||
}
|
||||
|
||||
#: The hosts a request or a redirect may land on for a given board host — a
|
||||
#: fixed API host for five of them, and the company's own subdomain for the
|
||||
#: two that put the slug in the hostname. Anything else is refused, at the
|
||||
#: first hop and at every redirect: a feed is external content, and a board
|
||||
#: row must never be able to point this fetch at an internal address.
|
||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
fixed = {
|
||||
"greenhouse": "boards-api.greenhouse.io",
|
||||
"lever": "api.lever.co",
|
||||
"ashby": "api.ashbyhq.com",
|
||||
"workable": "apply.workable.com",
|
||||
"smartrecruiters": "api.smartrecruiters.com",
|
||||
}
|
||||
if host in fixed:
|
||||
want = fixed[host]
|
||||
return lambda h: h == want
|
||||
if host == "recruitee":
|
||||
want = f"{slug}.recruitee.com"
|
||||
return lambda h: h == want
|
||||
if host == "personio":
|
||||
want = f"{slug}.jobs.personio.de"
|
||||
return lambda h: h == want
|
||||
return lambda h: False
|
||||
|
||||
|
||||
def _is_web_url(url: str) -> bool:
|
||||
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
||||
`javascript:` or `data:` link must never reach an href. Port of
|
||||
openings.go's `webURL`."""
|
||||
url = (url or "").strip()
|
||||
if not url:
|
||||
return False
|
||||
parsed = httpx.URL(url)
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
||||
|
||||
|
||||
class _Refused(Exception):
|
||||
"""A board answered somewhere its allow-list does not cover."""
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET") -> bytes:
|
||||
"""Fetch one board feed: https only, the host's own allow-list, at most
|
||||
three redirect hops each checked the same way, and the body cut off at
|
||||
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`."""
|
||||
allow = _allowed_host(host, slug)
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
parsed = httpx.URL(current)
|
||||
if parsed.scheme != "https" or not allow(parsed.host):
|
||||
raise _Refused(f"{current} is not an allowed host for {host}")
|
||||
response = client.request(
|
||||
method, current, timeout=TIMEOUT, follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
|
||||
json={} if method == "POST" else None,
|
||||
)
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
if not location:
|
||||
raise ValueError("redirect with no location")
|
||||
current = str(httpx.URL(current).join(location))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
body = response.content
|
||||
if len(body) > MAX_BYTES:
|
||||
raise ValueError(f"response too large ({len(body)} bytes)")
|
||||
return body
|
||||
raise ValueError("too many redirects")
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
||||
offset, `Z`, a bare date, epoch milliseconds, or "YYYY-MM-DD HH:MM:SS
|
||||
UTC". None for anything else, which `Item` reads as "now"."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, int | float):
|
||||
if value <= 0:
|
||||
return None
|
||||
seconds = value / 1000 if value > 10**12 else value
|
||||
return datetime.fromtimestamp(seconds, tz=UTC)
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
||||
if not match:
|
||||
return None
|
||||
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
# ------------------------------------------------------------- the parsers
|
||||
|
||||
#: What every parser turns a feed into: one dict per row, with these keys.
|
||||
Row = dict[str, Any]
|
||||
|
||||
|
||||
def _parse_greenhouse(body: bytes) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("jobs") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("absolute_url") or "").strip()
|
||||
if not title or not _is_web_url(url):
|
||||
continue
|
||||
departments = j.get("departments") or []
|
||||
# Greenhouse ships the description HTML-entity-escaped inside the
|
||||
# JSON string (`<p>`), so it needs unescaping before the tags
|
||||
# themselves can be stripped.
|
||||
content = html.unescape(j.get("content") or "")
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": ((j.get("location") or {}).get("name") or "").strip(),
|
||||
"url": url,
|
||||
"posted": _parse_dt(j.get("first_published") or j.get("updated_at")),
|
||||
"department": (departments[0].get("name") if departments else "") or "",
|
||||
"description": strip_html(content),
|
||||
"remote": None, # Greenhouse's feed does not say
|
||||
"salary": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_lever(body: bytes) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
rows = []
|
||||
for j in data:
|
||||
title = (j.get("text") or "").strip()
|
||||
url = (j.get("hostedUrl") or "").strip()
|
||||
if not title or not _is_web_url(url):
|
||||
continue
|
||||
categories = j.get("categories") or {}
|
||||
description = j.get("description") or j.get("descriptionPlain") or ""
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (categories.get("location") or "").strip(),
|
||||
"url": url,
|
||||
"posted": _parse_dt(j.get("createdAt")),
|
||||
"department": (categories.get("department") or categories.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": (j.get("workplaceType") or "").strip() or None,
|
||||
"salary": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_ashby(body: bytes) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("jobs") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
||||
if not title or not _is_web_url(url):
|
||||
continue
|
||||
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
||||
compensation = j.get("compensation") or {}
|
||||
salary = (compensation.get("compensationTierSummary")
|
||||
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (j.get("location") or "").strip(),
|
||||
"url": url,
|
||||
"posted": _parse_dt(j.get("publishedAt")),
|
||||
"department": (j.get("department") or j.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": j.get("isRemote"),
|
||||
"salary": salary,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_workable(body: bytes) -> list[Row]:
|
||||
"""https://apply.workable.com/api/v3/accounts/{slug}/jobs (POST, empty
|
||||
JSON body) -> `{"total": int, "results": [...]}`. The shape is confirmed
|
||||
live (2026-09-23); the fields are Workable's long-documented job object,
|
||||
since every slug this session tried on it happened to list nothing at
|
||||
the moment of fetching — see the plugin's docstring and the build report."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("results") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("url") or j.get("shortlink") or j.get("application_url") or "").strip()
|
||||
if not title or not _is_web_url(url):
|
||||
continue
|
||||
location = ", ".join(p for p in (j.get("city"), j.get("region"), j.get("country")) if p)
|
||||
description = " ".join(p for p in (
|
||||
j.get("description"), j.get("requirements"), j.get("benefits")) if p)
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or j.get("shortcode") or ""),
|
||||
"title": title,
|
||||
"location": location,
|
||||
"url": url,
|
||||
"posted": _parse_dt(j.get("published_on") or j.get("created_at")),
|
||||
"department": (j.get("department") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": j.get("telecommute"),
|
||||
"salary": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
|
||||
"""https://api.smartrecruiters.com/v1/companies/{slug}/postings ->
|
||||
`{"content": [...]}`. The list omits the job description (only the
|
||||
per-posting detail endpoint has it, and fetching one per row would be
|
||||
hundreds of extra requests per board), so `description` stays empty here."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("content") or []:
|
||||
title = (j.get("name") or "").strip()
|
||||
job_id = str(j.get("id") or "")
|
||||
if not title or not job_id:
|
||||
continue
|
||||
location = j.get("location") or {}
|
||||
loc_text = (location.get("fullLocation")
|
||||
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
||||
rows.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"location": (loc_text or "").strip(),
|
||||
"url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}",
|
||||
"posted": _parse_dt(j.get("releasedDate")),
|
||||
"department": ((j.get("department") or {}).get("label") or "").strip(),
|
||||
"description": "",
|
||||
"remote": location.get("remote"),
|
||||
"salary": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_recruitee(body: bytes) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("offers") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("careers_url") or "").strip()
|
||||
if not title or not _is_web_url(url):
|
||||
continue
|
||||
salary = j.get("salary") or {}
|
||||
salary_text = ""
|
||||
if isinstance(salary, dict) and (salary.get("min") or salary.get("max")):
|
||||
bounds = "-".join(str(v) for v in (salary.get("min"), salary.get("max")) if v)
|
||||
salary_text = f"{bounds} {salary.get('currency') or ''}".strip()
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (j.get("location") or j.get("city") or "").strip(),
|
||||
"url": url,
|
||||
"posted": _parse_dt(j.get("published_at") or j.get("created_at")),
|
||||
"department": (j.get("department") or "").strip(),
|
||||
"description": strip_html(j.get("description") or ""),
|
||||
"remote": j.get("remote"),
|
||||
"salary": salary_text,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_personio(body: bytes, slug: str) -> list[Row]:
|
||||
"""`{slug}.jobs.personio.de/xml` -> `<workzag-jobs><position>...`. No job
|
||||
URL in the feed itself; Personio's own page is `/job/{id}`, confirmed
|
||||
live against a real posting (2026-09-23)."""
|
||||
root = ET.fromstring(body)
|
||||
rows = []
|
||||
for position in root.findall("position"):
|
||||
job_id = (position.findtext("id") or "").strip()
|
||||
title = (position.findtext("name") or "").strip()
|
||||
if not job_id or not title:
|
||||
continue
|
||||
description = " ".join(
|
||||
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
||||
rows.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"location": (position.findtext("office") or "").strip(),
|
||||
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
|
||||
"posted": _parse_dt(position.findtext("createdAt")),
|
||||
"department": (position.findtext("department") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": "remote" in (position.findtext("office") or "").lower() or None,
|
||||
"salary": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
#: host -> parser. Every one takes the raw response body, plus the slug for
|
||||
#: the two hosts whose feed does not carry the URL or the company itself.
|
||||
_PARSERS: dict[str, Callable[..., list[Row]]] = {
|
||||
"greenhouse": lambda body, slug: _parse_greenhouse(body),
|
||||
"lever": lambda body, slug: _parse_lever(body),
|
||||
"ashby": lambda body, slug: _parse_ashby(body),
|
||||
"workable": lambda body, slug: _parse_workable(body),
|
||||
"smartrecruiters": _parse_smartrecruiters,
|
||||
"recruitee": lambda body, slug: _parse_recruitee(body),
|
||||
"personio": _parse_personio,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------- the classifier
|
||||
|
||||
#: The fixed set of functions, and the keyword rules that map a department
|
||||
#: (preferred) or a title to one of them. Ported verbatim, in order, from
|
||||
#: `cmd/seedgen/main.go`'s `rules` — first match wins, and "product" is
|
||||
#: genuinely listed twice: the first, narrower entry is checked before
|
||||
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
||||
#: matches the bare word "product", is checked only after all of those, so a
|
||||
#: title like "Data Product Manager" keeps classifying as data/product by the
|
||||
#: same rule order the Go tool used.
|
||||
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
||||
"customer success/support", "recruiting/people", "finance/legal/ops")
|
||||
|
||||
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("product", ("product manag", "product management", "product operations",
|
||||
"program manag", "technical program")),
|
||||
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
||||
"people team", "people operations", "human resources",
|
||||
" hr ", "hr ", "hrbp")),
|
||||
("design", ("design", "ux", "user experience", "creative")),
|
||||
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
||||
"seo", "social media", "demand gen", "events")),
|
||||
("customer success/support", ("customer success", "customer support",
|
||||
"customer experience", "customer service",
|
||||
"support", "success", "implementation",
|
||||
"onboarding", "professional services",
|
||||
"solutions consult", "technical account",
|
||||
"customer engineer", "deployment")),
|
||||
("sales", ("sales", "account executive", "account manager", "account director",
|
||||
"business development", "bdr", "sdr", "revenue", "partnership",
|
||||
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
||||
"pre-sales", "presales", "channel")),
|
||||
("data", ("data", "analytics", "machine learning", "ml ", " ml",
|
||||
"applied science", "research scientist", "statistic")),
|
||||
("product", ("product manag", "product management", "program manag",
|
||||
"product operations", "technical program", "product")),
|
||||
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
|
||||
"compliance", "operations", "ops", "administrat",
|
||||
"office", "procurement", "strategy", "corporate",
|
||||
"facilities", "risk", "audit", "tax", "treasury",
|
||||
"payroll", "executive assistant", "chief of staff",
|
||||
"workplace", "supply chain", "logistics", "fraud",
|
||||
"trust & safety", "trust and safety", "policy")),
|
||||
("engineering", ("engineer", "software", "developer", "infrastructure",
|
||||
"platform", "devops", "sre", "site reliability", "security",
|
||||
"information technology", " it", "it ", "technology",
|
||||
"technical staff", "hardware", "qa", "quality assurance",
|
||||
"architect", "research", "science", "scientist", "systems",
|
||||
"network", "cloud", "mobile", "frontend", "backend",
|
||||
"full stack", "fullstack", "web")),
|
||||
)
|
||||
|
||||
|
||||
def _classify_text(text: str) -> str:
|
||||
padded = f" {text.strip().lower()} "
|
||||
if not padded.strip():
|
||||
return ""
|
||||
for function, keywords in _RULES:
|
||||
if any(keyword in padded for keyword in keywords):
|
||||
return function
|
||||
return ""
|
||||
|
||||
|
||||
def classify(department: str, title: str) -> str:
|
||||
"""A department (preferred) or a title, to one of `FUNCTIONS`, or
|
||||
`"other"`. Port of seedgen's `classify`/`classifyText`."""
|
||||
return _classify_text(department or "") or _classify_text(title or "") or "other"
|
||||
|
||||
|
||||
def is_known_target(target: str) -> bool:
|
||||
"""`host:slug`, with `host` one of `_BOARDS` and a non-empty slug. What
|
||||
the target validator refuses: `roledesk companies add` and the pull job
|
||||
both call this before touching the network."""
|
||||
host, sep, slug = target.partition(":")
|
||||
return bool(sep) and host in _BOARDS and bool(slug.strip())
|
||||
|
||||
|
||||
def careers_url(target: str) -> str:
|
||||
"""The company's own careers page for `host:slug`, or "" for an unknown
|
||||
host. What `roledesk companies add` stores alongside the board."""
|
||||
host, _, slug = target.partition(":")
|
||||
if host not in _BOARDS:
|
||||
return ""
|
||||
return _BOARDS[host][1].format(slug=slug)
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
"""One board, fetched and parsed. Raises on any failure — refused host,
|
||||
timeout, a feed shape none of the parsers understand."""
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
host, _, slug = target.partition(":")
|
||||
url = _BOARDS[host][0].format(slug=slug)
|
||||
body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET")
|
||||
return _PARSERS[host](body, slug)
|
||||
|
||||
|
||||
def fetch_board(target: str) -> list[Row]:
|
||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||
"does this board answer, and what does it list". For `roledesk companies
|
||||
add|check`, which ask that question about one board at a time outside any
|
||||
pull. Raises the same way `_fetch_rows` does."""
|
||||
with httpx.Client() as client:
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
|
||||
class ATSSource(Source):
|
||||
name = "ats"
|
||||
description = "Company job boards: Greenhouse, Lever, Ashby, Workable, " \
|
||||
"SmartRecruiters, Recruitee, Personio."
|
||||
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
||||
#: company}` — `url` carries the `host:slug` target and `label` the
|
||||
#: company name, which is how `Source.targets()`'s generic dict reading
|
||||
#: hands both back without a plugin-specific override.
|
||||
subscription_field = "boards"
|
||||
|
||||
spec = PluginSpec(
|
||||
api_version=API_VERSION, id=name, version="1", label="ATS boards",
|
||||
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
|
||||
profiles={"default": Profile(
|
||||
kind="role",
|
||||
tasks=Tasks(
|
||||
summary=SummaryPolicy(enabled=False),
|
||||
tagging=TaggingPolicy(enabled=False),
|
||||
),
|
||||
presentation=ARTICLE_READER,
|
||||
)},
|
||||
)
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
client = httpx.Client()
|
||||
# Written by the pull job before any board is fetched (see
|
||||
# `roledesk/jobs/pull.py`); a plugin run on its own — a test, a dry
|
||||
# run — has no run to stamp itself against, so it makes one up.
|
||||
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
try:
|
||||
rows = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
continue
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
# `Store.mark_closed` compares against, not the board's own
|
||||
# bare job id.
|
||||
compound_id = f"{host}:{slug}:{row['source_id']}"
|
||||
seen_ids.append(compound_id)
|
||||
yield Item(
|
||||
source_id=compound_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": label,
|
||||
"place": row["location"],
|
||||
"department": row["department"],
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
"salary": row["salary"],
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
Vendored
+28
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"jobs": [
|
||||
{
|
||||
"id": "3001",
|
||||
"title": "Product Designer",
|
||||
"department": "Design",
|
||||
"team": "Product",
|
||||
"location": "Munich",
|
||||
"isRemote": false,
|
||||
"publishedAt": "2026-09-01T00:00:00Z",
|
||||
"jobUrl": "https://ashby.example/1",
|
||||
"applyUrl": "https://ashby.example/1/apply",
|
||||
"descriptionHtml": "<p>Design our <strong>onboarding</strong> flow.</p>",
|
||||
"compensation": {"compensationTierSummary": "70,000-90,000 EUR"}
|
||||
},
|
||||
{
|
||||
"id": "3002",
|
||||
"title": "Enterprise Account Executive | Germany",
|
||||
"department": "Sales",
|
||||
"team": "Account Executives",
|
||||
"location": "Berlin",
|
||||
"isRemote": true,
|
||||
"publishedAt": "2026-08-28T21:20:40.792+00:00",
|
||||
"jobUrl": "https://ashby.example/2",
|
||||
"descriptionHtml": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"jobs": [
|
||||
{
|
||||
"id": 1001,
|
||||
"title": "\t Senior Backend Engineer ",
|
||||
"location": {"name": " Berlin, Germany"},
|
||||
"absolute_url": "https://gh.example/1",
|
||||
"updated_at": "2026-09-10T12:00:00-04:00",
|
||||
"first_published": "2026-09-09T08:00:00-04:00",
|
||||
"departments": [{"id": 1, "name": "Engineering"}],
|
||||
"content": "<p>Build our <strong>core platform</strong>.</p>"
|
||||
},
|
||||
{
|
||||
"id": 1002,
|
||||
"title": "Account Executive",
|
||||
"location": {"name": "Phoenix, AZ"},
|
||||
"absolute_url": "https://gh.example/2",
|
||||
"updated_at": "2026-09-11T12:00:00-04:00",
|
||||
"departments": [{"id": 2, "name": "Sales"}],
|
||||
"content": ""
|
||||
},
|
||||
{
|
||||
"id": 1003,
|
||||
"title": "Evil posting",
|
||||
"location": {"name": "Anywhere"},
|
||||
"absolute_url": "javascript:alert(1)",
|
||||
"departments": []
|
||||
},
|
||||
{
|
||||
"id": 1004,
|
||||
"title": "Data posting",
|
||||
"location": {"name": "Anywhere"},
|
||||
"absolute_url": "data:text/html,hi",
|
||||
"departments": []
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+20
@@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"id": "2001",
|
||||
"text": "Staff Engineer, Platform",
|
||||
"hostedUrl": "https://lever.example/1",
|
||||
"createdAt": 1789430400000,
|
||||
"categories": {"location": "Remote", "department": "Engineering", "team": "Platform"},
|
||||
"workplaceType": "remote",
|
||||
"description": "<p>Own our <b>platform</b> reliability.</p>"
|
||||
},
|
||||
{
|
||||
"id": "2002",
|
||||
"text": "Customer Success Manager",
|
||||
"hostedUrl": "https://lever.example/2",
|
||||
"createdAt": 0,
|
||||
"categories": {"location": "New York", "department": "Customer Success"},
|
||||
"workplaceType": "on-site",
|
||||
"description": ""
|
||||
}
|
||||
]
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<workzag-jobs>
|
||||
|
||||
<position>
|
||||
<id>6001</id>
|
||||
<subcompany>Acme AG</subcompany>
|
||||
<office>Remote - Germany</office>
|
||||
<department>Sales</department>
|
||||
<recruitingCategory>Customer Success</recruitingCategory>
|
||||
<name>Business Development Representative</name>
|
||||
<jobDescriptions>
|
||||
<jobDescription>
|
||||
<name>Your mission</name>
|
||||
<value><![CDATA[<p>Grow our pipeline.</p>]]></value>
|
||||
</jobDescription>
|
||||
</jobDescriptions>
|
||||
<employmentType>permanent</employmentType>
|
||||
<seniority>entry-level</seniority>
|
||||
<schedule>full-time</schedule>
|
||||
<createdAt>2026-08-19T10:04:22+00:00</createdAt>
|
||||
</position>
|
||||
|
||||
<position>
|
||||
<id>6002</id>
|
||||
<subcompany>Acme AG</subcompany>
|
||||
<office>Munich</office>
|
||||
<department>Engineering</department>
|
||||
<name>Backend Engineer (all genders)</name>
|
||||
<jobDescriptions></jobDescriptions>
|
||||
<employmentType>permanent</employmentType>
|
||||
<seniority>experienced</seniority>
|
||||
<schedule>full-time</schedule>
|
||||
<createdAt>2026-09-21T14:53:35+00:00</createdAt>
|
||||
</position>
|
||||
|
||||
</workzag-jobs>
|
||||
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"offers": [
|
||||
{
|
||||
"id": 5001,
|
||||
"title": "Player Support Agent (German)",
|
||||
"slug": "player-support-agent-german-1",
|
||||
"careers_url": "https://acme.example/o/player-support-agent-german-1",
|
||||
"department": "Player Support",
|
||||
"city": "Berlin",
|
||||
"location": "Berlin",
|
||||
"remote": true,
|
||||
"on_site": false,
|
||||
"published_at": "2026-09-23 04:36:48 UTC",
|
||||
"created_at": "2026-09-20 04:27:06 UTC",
|
||||
"description": "<p>Support our <b>players</b>.</p>",
|
||||
"salary": {"min": 35000, "max": 45000, "currency": "EUR", "period": "year"}
|
||||
},
|
||||
{
|
||||
"id": 5002,
|
||||
"title": "Finance Manager",
|
||||
"slug": "finance-manager",
|
||||
"careers_url": "https://acme.example/o/finance-manager",
|
||||
"department": "Finance",
|
||||
"city": "Munich",
|
||||
"location": "Munich",
|
||||
"remote": false,
|
||||
"published_at": "2026-09-01 00:00:00 UTC",
|
||||
"description": "",
|
||||
"salary": {"min": null, "max": null, "currency": null, "period": null}
|
||||
}
|
||||
]
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"offset": 0,
|
||||
"limit": 100,
|
||||
"totalFound": 2,
|
||||
"content": [
|
||||
{
|
||||
"id": "744000151326819",
|
||||
"name": "Dir, Software Engrg Mgmt",
|
||||
"company": {"identifier": "Acme", "name": "Acme"},
|
||||
"releasedDate": "2026-09-23T08:47:37.279Z",
|
||||
"location": {
|
||||
"city": "Berlin",
|
||||
"country": "de",
|
||||
"remote": false,
|
||||
"hybrid": false,
|
||||
"fullLocation": "Berlin, Germany"
|
||||
},
|
||||
"department": {"id": "2294038", "label": "Engineering, Infrastructure and Operations"},
|
||||
"function": {"id": "engineering", "label": "Engineering"}
|
||||
},
|
||||
{
|
||||
"id": "744000147485112",
|
||||
"name": "Sales Enablement Manager",
|
||||
"company": {"identifier": "Acme", "name": "Acme"},
|
||||
"releasedDate": "2026-09-10T09:00:00.000Z",
|
||||
"location": {"city": "", "country": "", "remote": true, "fullLocation": "Remote"},
|
||||
"department": {"id": "1", "label": "Sales"},
|
||||
"function": {"id": "sales", "label": "Sales"}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"total": 2,
|
||||
"results": [
|
||||
{
|
||||
"id": "4001",
|
||||
"title": "Backend Engineer",
|
||||
"department": "Engineering",
|
||||
"city": "Berlin",
|
||||
"region": "",
|
||||
"country": "Germany",
|
||||
"telecommute": true,
|
||||
"published_on": "2026-09-01",
|
||||
"created_at": "2026-08-30T10:00:00.000Z",
|
||||
"url": "https://workable.example/1",
|
||||
"shortlink": "https://workable.example/short/1",
|
||||
"description": "<p>Ship our API.</p>",
|
||||
"requirements": "<p>5+ years.</p>",
|
||||
"benefits": ""
|
||||
},
|
||||
{
|
||||
"id": "4002",
|
||||
"shortcode": "ABCDEF",
|
||||
"title": "Account Manager",
|
||||
"department": "Sales",
|
||||
"city": "Munich",
|
||||
"region": "",
|
||||
"country": "Germany",
|
||||
"telecommute": false,
|
||||
"published_on": "2026-08-15",
|
||||
"url": "https://workable.example/2",
|
||||
"description": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
"""The `ats` plugin: each host's parser against a saved feed, the function
|
||||
classifier, and the fetch allow-list. No network — every fixture is a file
|
||||
under `tests/fixtures/ats/`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
|
||||
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
||||
body = path.read_bytes()
|
||||
return ats._PARSERS[host](body, slug)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ hosts
|
||||
|
||||
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
||||
rows = _rows("greenhouse")
|
||||
# The javascript: and data: rows are refused, same as the two real ones
|
||||
# are kept: a feed is external content, never trusted as a URL.
|
||||
assert [r["source_id"] for r in rows] == ["1001", "1002"]
|
||||
first = rows[0]
|
||||
assert first["title"] == "Senior Backend Engineer" # trimmed
|
||||
assert first["location"] == "Berlin, Germany"
|
||||
assert first["department"] == "Engineering"
|
||||
assert first["description"] == "Build our core platform."
|
||||
assert first["posted"] is not None and first["posted"].year == 2026
|
||||
assert first["remote"] is None
|
||||
|
||||
|
||||
def test_lever_parses_categories_and_workplace_type() -> None:
|
||||
rows = _rows("lever")
|
||||
assert [r["source_id"] for r in rows] == ["2001", "2002"]
|
||||
first = rows[0]
|
||||
assert first["department"] == "Engineering"
|
||||
assert first["remote"] == "remote"
|
||||
assert first["description"] == "Own our platform reliability."
|
||||
# createdAt=0 means "no date", not epoch zero.
|
||||
assert rows[1]["posted"] is None
|
||||
|
||||
|
||||
def test_ashby_parses_is_remote_and_compensation() -> None:
|
||||
rows = _rows("ashby")
|
||||
assert [r["source_id"] for r in rows] == ["3001", "3002"]
|
||||
assert rows[0]["remote"] is False
|
||||
assert rows[0]["salary"] == "70,000-90,000 EUR"
|
||||
assert rows[1]["remote"] is True
|
||||
|
||||
|
||||
def test_workable_parses_location_parts_and_telecommute() -> None:
|
||||
rows = _rows("workable")
|
||||
assert [r["source_id"] for r in rows] == ["4001", "4002"]
|
||||
assert rows[0]["location"] == "Berlin, Germany"
|
||||
assert rows[0]["remote"] is True
|
||||
assert "Ship our API." in rows[0]["description"]
|
||||
|
||||
|
||||
def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
|
||||
rows = _rows("smartrecruiters", slug="Acme")
|
||||
assert [r["source_id"] for r in rows] == ["744000151326819", "744000147485112"]
|
||||
assert rows[0]["url"] == "https://jobs.smartrecruiters.com/Acme/744000151326819"
|
||||
assert rows[0]["location"] == "Berlin, Germany"
|
||||
assert rows[1]["remote"] is True
|
||||
|
||||
|
||||
def test_recruitee_parses_salary_bounds_and_careers_url() -> None:
|
||||
rows = _rows("recruitee")
|
||||
assert [r["source_id"] for r in rows] == ["5001", "5002"]
|
||||
assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1"
|
||||
assert rows[0]["salary"] == "35000-45000 EUR"
|
||||
assert rows[1]["salary"] == ""
|
||||
|
||||
|
||||
def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
|
||||
rows = _rows("personio", slug="acme")
|
||||
assert [r["source_id"] for r in rows] == ["6001", "6002"]
|
||||
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
|
||||
assert rows[0]["description"] == "Grow our pipeline."
|
||||
assert rows[0]["remote"] is True # "Remote - Germany"
|
||||
assert rows[1]["remote"] is None # "Munich"
|
||||
|
||||
|
||||
# --------------------------------------------------------------- classify
|
||||
|
||||
def test_classify_matches_department_before_title() -> None:
|
||||
assert ats.classify("Engineering", "Marketing Lead") == "engineering"
|
||||
assert ats.classify("", "Senior Backend Engineer") == "engineering"
|
||||
assert ats.classify("", "Product Manager, Payments") == "product"
|
||||
assert ats.classify("", "Data Product Manager") == "product"
|
||||
assert ats.classify("", "Machine Learning Engineer") == "data"
|
||||
assert ats.classify("", "Executive Assistant to the CEO") == "finance/legal/ops"
|
||||
assert ats.classify("", "") == "other"
|
||||
|
||||
|
||||
# ------------------------------------------------------------- allow-list
|
||||
|
||||
def test_is_known_target_accepts_the_seven_hosts_and_refuses_others() -> None:
|
||||
for host in ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||
"recruitee", "personio"):
|
||||
assert ats.is_known_target(f"{host}:acme")
|
||||
assert not ats.is_known_target("linkedin:acme")
|
||||
assert not ats.is_known_target("workday:acme")
|
||||
assert not ats.is_known_target("greenhouse:")
|
||||
assert not ats.is_known_target("nohostnoseparator")
|
||||
|
||||
|
||||
def test_allowed_host_refuses_a_foreign_host_per_board() -> None:
|
||||
allow = ats._allowed_host("greenhouse", "acme")
|
||||
assert allow("boards-api.greenhouse.io")
|
||||
assert not allow("boards-api.greenhouse.io.evil.example")
|
||||
assert not allow("api.lever.co")
|
||||
|
||||
# recruitee and personio key off the company's own subdomain, so one
|
||||
# company's slug must not pass for another's.
|
||||
allow_recruitee = ats._allowed_host("recruitee", "acme")
|
||||
assert allow_recruitee("acme.recruitee.com")
|
||||
assert not allow_recruitee("other.recruitee.com")
|
||||
|
||||
|
||||
def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
|
||||
assert ats.careers_url("greenhouse:acme") == "https://job-boards.greenhouse.io/acme"
|
||||
assert ats.careers_url("ashby:acme") == "https://jobs.ashbyhq.com/acme"
|
||||
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
|
||||
assert ats.careers_url("workday:acme") == ""
|
||||
Reference in new issue
Block a user