A second pull (schedule, Run button, `app job pull` in another process) could start while one was fetching and overwrite the stored run marker, so the first run's boards failed their stamp check or a settle closed on the other run's id lists. `claim_run` sets `pull:run` in one conditional statement, only while it equals `pull:settled` or is older than an hour; a refused `pull` job queues no stage and says so in its result and a notice, and the CLI exits 1. Batches carry the marker in their payload, the plugins stamp with it, and the settle reads it off its own batches' payloads before releasing the run. Co-Authored-By: Claude Opus <noreply@anthropic.com>
710 lines
32 KiB
Python
710 lines
32 KiB
Python
"""ATS boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio.
|
|
|
|
Every one of these hosts is a company's own careers page speaking public,
|
|
unauthenticated JSON (Personio: XML) — first-party, no scraping, and a role
|
|
that drops off the board is *closed*, which is the cheapest staleness check
|
|
there is.
|
|
|
|
The field map, the redirect allow-list and the 10 s timeout are ported from
|
|
`openings.go` and `cmd/seedgen/main.go` in
|
|
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
|
`classify`'s nine functions and their keywords. Workable, SmartRecruiters,
|
|
Recruitee and Personio are new: jobhunt-hq only read Greenhouse, Lever and
|
|
Ashby, so those four are written from one real board of each host, fetched
|
|
while this file was written (2026-09-23; Workable on 2026-09-28, when the
|
|
first board with roles was seen), rather than from the Go source.
|
|
|
|
The body cap is not a straight port — see `MAX_BYTES` below.
|
|
|
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|
`lever:acme`. Every board is fetched independently; one that fails or times
|
|
out is logged, recorded as failing under `ats:failing:<target>` (see
|
|
`ATSSource.fetch`), and skipped, and the rest continue.
|
|
|
|
Workable and SmartRecruiters page their list and state a total; every other
|
|
host answers with the whole board at once. A paged board counts as complete
|
|
only when it read as many distinct roles as the total says (see `_paged`).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: Matches openings.go: a board answers in 10s or it is skipped. openings.go's
|
|
#: 8 MB body cap does not: it was written against arbitrary URLs, and every
|
|
#: URL fetched here is `?content=true` against a fixed allow-list of seven ATS
|
|
#: API hosts. The real run of 2026-09-23 hit it on the biggest employers —
|
|
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
|
|
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
|
|
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
|
|
#: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least
|
|
#: the titles land.
|
|
TIMEOUT = 10.0
|
|
MAX_BYTES = 96 << 20
|
|
|
|
#: host -> (feed URL template, careers URL template). `{slug}` is filled in;
|
|
#: recruitee and personio put the slug in the hostname instead of the path.
|
|
_BOARDS: dict[str, tuple[str, str]] = {
|
|
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs?content=true",
|
|
"https://job-boards.greenhouse.io/{slug}"),
|
|
"lever": ("https://api.lever.co/v0/postings/{slug}?mode=json",
|
|
"https://jobs.lever.co/{slug}"),
|
|
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}",
|
|
"https://jobs.ashbyhq.com/{slug}"),
|
|
"workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs",
|
|
"https://apply.workable.com/{slug}"),
|
|
"smartrecruiters": ("https://api.smartrecruiters.com/v1/companies/{slug}/postings",
|
|
"https://jobs.smartrecruiters.com/{slug}"),
|
|
"recruitee": ("https://{slug}.recruitee.com/api/offers/",
|
|
"https://{slug}.recruitee.com"),
|
|
"personio": ("https://{slug}.jobs.personio.de/xml",
|
|
"https://{slug}.jobs.personio.de"),
|
|
}
|
|
|
|
#: The hosts a request or a redirect may land on for a given board host — a
|
|
#: fixed API host for five of them, and the company's own subdomain for the
|
|
#: two that put the slug in the hostname. Anything else is refused, at the
|
|
#: first hop and at every redirect: a feed is external content, and a board
|
|
#: row must never be able to point this fetch at an internal address.
|
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
|
fixed = {
|
|
"greenhouse": "boards-api.greenhouse.io",
|
|
"lever": "api.lever.co",
|
|
"ashby": "api.ashbyhq.com",
|
|
"workable": "apply.workable.com",
|
|
"smartrecruiters": "api.smartrecruiters.com",
|
|
}
|
|
if host in fixed:
|
|
want = fixed[host]
|
|
return lambda h: h == want
|
|
if host == "recruitee":
|
|
want = f"{slug}.recruitee.com"
|
|
return lambda h: h == want
|
|
if host == "personio":
|
|
want = f"{slug}.jobs.personio.de"
|
|
return lambda h: h == want
|
|
return lambda h: False
|
|
|
|
|
|
def _is_web_url(url: str) -> bool:
|
|
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
|
`javascript:` or `data:` link must never reach an href. Port of
|
|
openings.go's `webURL`."""
|
|
url = (url or "").strip()
|
|
if not url:
|
|
return False
|
|
parsed = httpx.URL(url)
|
|
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
|
|
|
|
|
class _Refused(Exception):
|
|
"""A board answered somewhere its allow-list does not cover."""
|
|
|
|
|
|
class _TooLarge(ValueError):
|
|
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
|
|
knows the old shape still catches it; `_fetch_rows` catches it by type to
|
|
retry Greenhouse without `?content=true`."""
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
|
method: str = "GET", payload: dict[str, Any] | None = None) -> bytes:
|
|
"""Fetch one board feed: https only, the host's own allow-list, at most
|
|
three redirect hops each checked the same way, and the body cut off at
|
|
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
|
sends `payload` as JSON, `{}` when there is none."""
|
|
allow = _allowed_host(host, slug)
|
|
current = url
|
|
for _hop in range(4):
|
|
parsed = httpx.URL(current)
|
|
if parsed.scheme != "https" or not allow(parsed.host):
|
|
raise _Refused(f"{current} is not an allowed host for {host}")
|
|
response = client.request(
|
|
method, current, timeout=TIMEOUT, follow_redirects=False,
|
|
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
|
|
json=(payload or {}) if method == "POST" else None,
|
|
)
|
|
if response.is_redirect:
|
|
location = response.headers.get("location") or ""
|
|
if not location:
|
|
raise ValueError("redirect with no location")
|
|
current = str(httpx.URL(current).join(location))
|
|
continue
|
|
response.raise_for_status()
|
|
body = response.content
|
|
if len(body) > MAX_BYTES:
|
|
raise _TooLarge(f"response too large ({len(body)} bytes)")
|
|
return body
|
|
raise ValueError("too many redirects")
|
|
|
|
|
|
def _parse_dt(value: Any) -> datetime | None:
|
|
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
|
offset, `Z`, a bare date, epoch milliseconds, or "YYYY-MM-DD HH:MM:SS
|
|
UTC". None for anything else, which `Item` reads as "now"."""
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, int | float):
|
|
if value <= 0:
|
|
return None
|
|
seconds = value / 1000 if value > 10**12 else value
|
|
return datetime.fromtimestamp(seconds, tz=UTC)
|
|
text = str(value).strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
|
if not match:
|
|
return None
|
|
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
# ------------------------------------------------------------- the parsers
|
|
|
|
#: What every parser turns a feed into: one dict per row, with these keys.
|
|
Row = dict[str, Any]
|
|
|
|
|
|
def _parse_greenhouse(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("jobs") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("absolute_url") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
departments = j.get("departments") or []
|
|
# Greenhouse ships the description HTML-entity-escaped inside the
|
|
# JSON string (`<p>`), so it needs unescaping before the tags
|
|
# themselves can be stripped.
|
|
content = html.unescape(j.get("content") or "")
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": ((j.get("location") or {}).get("name") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("first_published") or j.get("updated_at")),
|
|
"department": (departments[0].get("name") if departments else "") or "",
|
|
"description": strip_html(content),
|
|
"remote": None, # Greenhouse's feed does not say
|
|
"salary": "",
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_lever(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
if not isinstance(data, list):
|
|
return []
|
|
rows = []
|
|
for j in data:
|
|
title = (j.get("text") or "").strip()
|
|
url = (j.get("hostedUrl") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
categories = j.get("categories") or {}
|
|
description = j.get("description") or j.get("descriptionPlain") or ""
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (categories.get("location") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("createdAt")),
|
|
"department": (categories.get("department") or categories.get("team") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": (j.get("workplaceType") or "").strip() or None,
|
|
"salary": "",
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_ashby(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("jobs") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
|
compensation = j.get("compensation") or {}
|
|
salary = (compensation.get("compensationTierSummary")
|
|
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (j.get("location") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("publishedAt")),
|
|
"department": (j.get("department") or j.get("team") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": j.get("isRemote"),
|
|
"salary": salary,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
|
return _workable_rows(json.loads(body), slug)
|
|
|
|
|
|
def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
|
|
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
|
|
Face's board answered on 2026-09-28. Ten results a page; the next page is
|
|
the same POST with `{"token": nextPage}`, and the last page has no
|
|
`nextPage` (see `_paged`). A result carries no URL and no text: the job page is
|
|
`https://apply.workable.com/{slug}/j/{shortcode}/`, and the description
|
|
is only on the per-job endpoint, one request per role, so `description`
|
|
stays empty here as it does for SmartRecruiters. `location` is an object
|
|
and `department` a list; `remote` is a flag beside `workplace`
|
|
(remote, hybrid, on_site)."""
|
|
rows = []
|
|
for j in data.get("results") or []:
|
|
title = (j.get("title") or "").strip()
|
|
shortcode = (j.get("shortcode") or "").strip()
|
|
if not title or not shortcode:
|
|
continue
|
|
loc = j.get("location") or {}
|
|
location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p)
|
|
dept = j.get("department") or ""
|
|
if isinstance(dept, list):
|
|
dept = ", ".join(d for d in dept if d)
|
|
rows.append({
|
|
"source_id": str(j.get("id") or shortcode),
|
|
"title": title,
|
|
"location": location,
|
|
"url": f"https://apply.workable.com/{slug}/j/{shortcode}/",
|
|
"posted": _parse_dt(j.get("published")),
|
|
"department": dept.strip(),
|
|
"description": "",
|
|
"remote": bool(j.get("remote")) or j.get("workplace") == "remote",
|
|
"salary": "",
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
|
|
return _smartrecruiters_rows(json.loads(body), slug)
|
|
|
|
|
|
def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
|
|
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
|
|
page, whatever `limit` asks for; the next is `?offset=` further on (see
|
|
`_paged`). The list omits the job description (only the
|
|
per-posting detail endpoint has it, and fetching one per row would be
|
|
hundreds of extra requests per board), so `description` stays empty here."""
|
|
rows = []
|
|
for j in data.get("content") or []:
|
|
title = (j.get("name") or "").strip()
|
|
job_id = str(j.get("id") or "")
|
|
if not title or not job_id:
|
|
continue
|
|
location = j.get("location") or {}
|
|
loc_text = (location.get("fullLocation")
|
|
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
|
rows.append({
|
|
"source_id": job_id,
|
|
"title": title,
|
|
"location": (loc_text or "").strip(),
|
|
"url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}",
|
|
"posted": _parse_dt(j.get("releasedDate")),
|
|
"department": ((j.get("department") or {}).get("label") or "").strip(),
|
|
"description": "",
|
|
"remote": location.get("remote"),
|
|
"salary": "",
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_recruitee(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("offers") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("careers_url") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
salary = j.get("salary") or {}
|
|
salary_text = ""
|
|
if isinstance(salary, dict) and (salary.get("min") or salary.get("max")):
|
|
bounds = "-".join(str(v) for v in (salary.get("min"), salary.get("max")) if v)
|
|
salary_text = f"{bounds} {salary.get('currency') or ''}".strip()
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (j.get("location") or j.get("city") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("published_at") or j.get("created_at")),
|
|
"department": (j.get("department") or "").strip(),
|
|
"description": strip_html(j.get("description") or ""),
|
|
"remote": j.get("remote"),
|
|
"salary": salary_text,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_personio(body: bytes, slug: str) -> list[Row]:
|
|
"""`{slug}.jobs.personio.de/xml` -> `<workzag-jobs><position>...`. No job
|
|
URL in the feed itself; Personio's own page is `/job/{id}`, confirmed
|
|
live against a real posting (2026-09-23)."""
|
|
root = ET.fromstring(body)
|
|
rows = []
|
|
for position in root.findall("position"):
|
|
job_id = (position.findtext("id") or "").strip()
|
|
title = (position.findtext("name") or "").strip()
|
|
if not job_id or not title:
|
|
continue
|
|
description = " ".join(
|
|
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
|
rows.append({
|
|
"source_id": job_id,
|
|
"title": title,
|
|
"location": (position.findtext("office") or "").strip(),
|
|
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
|
|
"posted": _parse_dt(position.findtext("createdAt")),
|
|
"department": (position.findtext("department") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": "remote" in (position.findtext("office") or "").lower() or None,
|
|
"salary": "",
|
|
})
|
|
return rows
|
|
|
|
|
|
#: host -> parser. Every one takes the raw response body, plus the slug for
|
|
#: the two hosts whose feed does not carry the URL or the company itself.
|
|
_PARSERS: dict[str, Callable[..., list[Row]]] = {
|
|
"greenhouse": lambda body, slug: _parse_greenhouse(body),
|
|
"lever": lambda body, slug: _parse_lever(body),
|
|
"ashby": lambda body, slug: _parse_ashby(body),
|
|
"workable": _parse_workable,
|
|
"smartrecruiters": _parse_smartrecruiters,
|
|
"recruitee": lambda body, slug: _parse_recruitee(body),
|
|
"personio": _parse_personio,
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------- the classifier
|
|
|
|
#: The fixed set of functions, and the keyword rules that map a department
|
|
#: (preferred) or a title to one of them. Ported verbatim, in order, from
|
|
#: `cmd/seedgen/main.go`'s `rules` — first match wins, and "product" is
|
|
#: genuinely listed twice: the first, narrower entry is checked before
|
|
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
|
#: matches the bare word "product", is checked only after all of those, so a
|
|
#: title like "Data Product Manager" keeps classifying as data/product by the
|
|
#: same rule order the Go tool used.
|
|
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
|
"customer success/support", "recruiting/people", "finance/legal/ops")
|
|
|
|
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
("product", ("product manag", "product management", "product operations",
|
|
"program manag", "technical program")),
|
|
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
|
"people team", "people operations", "human resources",
|
|
" hr ", "hr ", "hrbp")),
|
|
("design", ("design", "ux", "user experience", "creative")),
|
|
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
|
"seo", "social media", "demand gen", "events")),
|
|
("customer success/support", ("customer success", "customer support",
|
|
"customer experience", "customer service",
|
|
"support", "success", "implementation",
|
|
"onboarding", "professional services",
|
|
"solutions consult", "technical account",
|
|
"customer engineer", "deployment")),
|
|
("sales", ("sales", "account executive", "account manager", "account director",
|
|
"business development", "bdr", "sdr", "revenue", "partnership",
|
|
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
|
"pre-sales", "presales", "channel")),
|
|
("data", ("data", "analytics", "machine learning", "ml ", " ml",
|
|
"applied science", "research scientist", "statistic")),
|
|
("product", ("product manag", "product management", "program manag",
|
|
"product operations", "technical program", "product")),
|
|
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
|
|
"compliance", "operations", "ops", "administrat",
|
|
"office", "procurement", "strategy", "corporate",
|
|
"facilities", "risk", "audit", "tax", "treasury",
|
|
"payroll", "executive assistant", "chief of staff",
|
|
"workplace", "supply chain", "logistics", "fraud",
|
|
"trust & safety", "trust and safety", "policy")),
|
|
("engineering", ("engineer", "software", "developer", "infrastructure",
|
|
"platform", "devops", "sre", "site reliability", "security",
|
|
"information technology", " it", "it ", "technology",
|
|
"technical staff", "hardware", "qa", "quality assurance",
|
|
"architect", "research", "science", "scientist", "systems",
|
|
"network", "cloud", "mobile", "frontend", "backend",
|
|
"full stack", "fullstack", "web")),
|
|
)
|
|
|
|
|
|
def _classify_text(text: str) -> str:
|
|
padded = f" {text.strip().lower()} "
|
|
if not padded.strip():
|
|
return ""
|
|
for function, keywords in _RULES:
|
|
if any(keyword in padded for keyword in keywords):
|
|
return function
|
|
return ""
|
|
|
|
|
|
def classify(department: str, title: str) -> str:
|
|
"""A department (preferred) or a title, to one of `FUNCTIONS`, or
|
|
`"other"`. Port of seedgen's `classify`/`classifyText`."""
|
|
return _classify_text(department or "") or _classify_text(title or "") or "other"
|
|
|
|
|
|
def is_known_target(target: str) -> bool:
|
|
"""`host:slug`, with `host` one of `_BOARDS` and a non-empty slug. What
|
|
the target validator refuses: `app companies add` and the pull job
|
|
both call this before touching the network."""
|
|
host, sep, slug = target.partition(":")
|
|
return bool(sep) and host in _BOARDS and bool(slug.strip())
|
|
|
|
|
|
def careers_url(target: str) -> str:
|
|
"""The company's own careers page for `host:slug`, or "" for an unknown
|
|
host. What `app companies add` stores alongside the board."""
|
|
host, _, slug = target.partition(":")
|
|
if host not in _BOARDS:
|
|
return ""
|
|
return _BOARDS[host][1].format(slug=slug)
|
|
|
|
|
|
#: Pages read from one paged board at most, and the pause between two of
|
|
#: them. Workable answers ten roles a page and SmartRecruiters a hundred, so
|
|
#: the cap is 1,000 and 10,000 roles; the pause keeps Workable, which answers
|
|
#: a burst with 429, from seeing one.
|
|
MAX_PAGES = 100
|
|
PAGE_DELAY = 0.5
|
|
|
|
#: SmartRecruiters' page size, which is also the most it serves per page.
|
|
SMARTRECRUITERS_PAGE = 100
|
|
|
|
#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its
|
|
#: raw results, the total it states, and the cursor of the next page (None
|
|
#: after the last). The first call gets cursor None.
|
|
Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any]
|
|
|
|
|
|
def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
|
url = _BOARDS["workable"][0].format(slug=slug)
|
|
data = json.loads(_get(client, url, "workable", slug, method="POST",
|
|
payload={"token": cursor} if cursor else {}))
|
|
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
|
|
|
|
|
|
def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
|
offset = int(cursor or 0)
|
|
url = (_BOARDS["smartrecruiters"][0].format(slug=slug)
|
|
+ f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}")
|
|
data = json.loads(_get(client, url, "smartrecruiters", slug))
|
|
content = data.get("content") or []
|
|
return data, content, data.get("totalFound"), offset + len(content) if content else None
|
|
|
|
|
|
#: host -> (one page, that page's rows) for the hosts that page.
|
|
_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page],
|
|
Callable[[dict[str, Any], str], list[Row]]]] = {
|
|
"workable": (_workable_page, _workable_rows),
|
|
"smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows),
|
|
}
|
|
|
|
|
|
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|
"""Every page of a Workable or SmartRecruiters board, `PAGE_DELAY` apart,
|
|
until the host has no next page, the stated total is reached, or
|
|
`MAX_PAGES`. Returns the rows and whether the board is complete: as many
|
|
distinct results read as the host's stated total. Counted on the raw
|
|
results, so a row the parser drops does not make a board incomplete, and
|
|
on distinct ids, so a page that shifted under an offset and repeated a
|
|
role does not make it complete.
|
|
|
|
A 429 after the first page stops the read with what it has, incomplete;
|
|
one on the first page raises like any other failure."""
|
|
host, _, slug = target.partition(":")
|
|
page_of, rows_of = _PAGED[host]
|
|
rows: dict[str, Row] = {}
|
|
seen: set[str] = set()
|
|
total: Any = None
|
|
cursor: Any = None
|
|
for page in range(MAX_PAGES):
|
|
if page:
|
|
time.sleep(PAGE_DELAY)
|
|
try:
|
|
data, results, total, cursor = page_of(client, slug, cursor)
|
|
except httpx.HTTPStatusError as exc:
|
|
if not page or exc.response.status_code != 429:
|
|
raise
|
|
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
|
|
target, page + 1, len(seen), total)
|
|
return list(rows.values()), False
|
|
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
|
for row in rows_of(data, slug):
|
|
rows.setdefault(row["source_id"], row)
|
|
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
|
break
|
|
complete = isinstance(total, int) and len(seen) == total
|
|
if not complete:
|
|
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), total)
|
|
return list(rows.values()), complete
|
|
|
|
|
|
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|
"""One board, fetched and parsed: its rows and whether they are the whole
|
|
board. Raises on any failure — refused host, timeout, a feed shape none
|
|
of the parsers understand. A host that does not page answers with the
|
|
whole board, so it is complete whenever it parses; a paged one is
|
|
complete when `_paged` says so.
|
|
|
|
A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES`
|
|
gets one retry without it: the description is lost, but the titles land
|
|
rather than the whole board being skipped."""
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
host, _, slug = target.partition(":")
|
|
if host in _PAGED:
|
|
return _paged(client, target)
|
|
url = _BOARDS[host][0].format(slug=slug)
|
|
try:
|
|
body = _get(client, url, host, slug, method="GET")
|
|
except _TooLarge:
|
|
if host != "greenhouse" or "?" not in url:
|
|
raise
|
|
log.warning("ats: %s: body exceeded %d bytes with content=true; "
|
|
"retrying without it", target, MAX_BYTES)
|
|
url = url.split("?", 1)[0]
|
|
body = _get(client, url, host, slug, method="GET")
|
|
return _PARSERS[host](body, slug), True
|
|
|
|
|
|
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
|
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
|
"does this board answer, what does it list, and is that all of it". For
|
|
`app companies add|check`, which ask that question about one board at a
|
|
time outside any pull. Raises the same way `_fetch_rows` does."""
|
|
with httpx.Client() as client:
|
|
return _fetch_rows(client, target)
|
|
|
|
|
|
def failure_reason(exc: BaseException) -> str:
|
|
"""One short line for why a board did not answer: the status of an HTTP
|
|
error, else the exception's class and the first line of its message."""
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
|
lines = str(exc).strip().splitlines()
|
|
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
|
return text[:200]
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class ATSSource(Source):
|
|
name = "ats"
|
|
description = "Company job boards: Greenhouse, Lever, Ashby, Workable, " \
|
|
"SmartRecruiters, Recruitee, Personio."
|
|
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
|
#: company}` — `url` carries the `host:slug` target and `label` the
|
|
#: company name, which is how `Source.targets()`'s generic dict reading
|
|
#: hands both back without a plugin-specific override.
|
|
subscription_field = "boards"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="ATS boards",
|
|
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# Handed in by the batch that runs this plugin, as the marker of
|
|
# the pull run that queued it (see `app/jobs/pull.py`), and never
|
|
# read from state, where a later run may have replaced it. A
|
|
# plugin run on its own — a test, a dry run — has no run to stamp
|
|
# itself against, so it makes one up.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
|
|
for target, label in self.targets():
|
|
host, _, slug = target.partition(":")
|
|
failing = f"ats:failing:{target}"
|
|
try:
|
|
rows, complete = _fetch_rows(client, target)
|
|
except Exception as exc:
|
|
log.warning("ats: %s failed (%s)", target, exc)
|
|
# The first failure in a row sets `since`, later ones only
|
|
# the reason; `app.jobs.pull` closes the board's roles
|
|
# once `since` is `STALE_DAYS` old.
|
|
try:
|
|
since = json.loads(self.state.get(failing, "") or "{}").get("since")
|
|
except ValueError:
|
|
since = None
|
|
self.state[failing] = json.dumps({
|
|
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
|
"reason": failure_reason(exc)})
|
|
continue
|
|
self.state.pop(failing, None)
|
|
|
|
seen_ids = []
|
|
for row in rows:
|
|
# The compound id, matching `roles.source_id` — what
|
|
# `Store.mark_closed` compares against, not the board's own
|
|
# bare job id.
|
|
compound_id = f"{host}:{slug}:{row['source_id']}"
|
|
seen_ids.append(compound_id)
|
|
yield Item(
|
|
source_id=compound_id,
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=(row["description"] or "")[:300],
|
|
body=row["description"],
|
|
published_at=row["posted"],
|
|
subscription=label,
|
|
kind="role",
|
|
raw={
|
|
"company": label,
|
|
"place": row["location"],
|
|
"department": row["department"],
|
|
"function": classify(row["department"], row["title"]),
|
|
"remote": row["remote"],
|
|
"salary": row["salary"],
|
|
"board": host,
|
|
"slug": slug,
|
|
},
|
|
)
|
|
# Stamped only once the board has parsed without raising, so a
|
|
# board that blew up half way through is neither marked ok nor
|
|
# has a partial id list treated as the whole of it. `complete`
|
|
# says whether the ids are the whole board, which the pull
|
|
# needs before it closes what they leave out.
|
|
self.state[f"ats:ok:{target}"] = run_marker
|
|
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
|
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|