1193 lines
56 KiB
Python
1193 lines
56 KiB
Python
"""ATS boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio,
|
|
Teamtailor.
|
|
|
|
Every one of these hosts is a company's own careers page speaking public,
|
|
unauthenticated JSON (Personio, Teamtailor: XML) — first-party, no scraping, and a role
|
|
that drops off the board is *closed*, which is the cheapest staleness check
|
|
there is.
|
|
|
|
The field map and the redirect allow-list are ported from
|
|
`openings.go` and `cmd/seedgen/main.go` in
|
|
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
|
`classify`'s nine functions and their keywords. Workable, SmartRecruiters,
|
|
Recruitee and Personio are new: jobhunt-hq only read Greenhouse, Lever and
|
|
Ashby, so those four are written from one real board of each host, fetched
|
|
while this file was written (2026-09-23; Workable on 2026-09-28, when the
|
|
first board with roles was seen), rather than from the Go source. Teamtailor
|
|
is the same, written from live feeds on 2026-09-29.
|
|
|
|
The body cap is not a straight port — see `MAX_BYTES` below.
|
|
|
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|
`lever:acme`, `teamtailor:acme`. Every board is fetched independently; one that fails or times
|
|
out is logged, recorded as failing under `ats:failing:<target>` (see
|
|
`ATSSource.fetch`), and skipped, and the rest continue.
|
|
|
|
A Teamtailor slug is the `acme` of acme.teamtailor.com, or, for a career site
|
|
on the company's own domain, that whole host name: `teamtailor:careers.acme.com`
|
|
(see `_teamtailor_site`).
|
|
|
|
Workable and SmartRecruiters page their list and state a total; Teamtailor
|
|
pages by offset and states none; every other host answers with the whole board
|
|
at once. A board with a total counts as complete only when it read as many
|
|
distinct roles as the total says, one without when it read up to an empty page
|
|
(see `_paged`).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every
|
|
#: URL fetched here is `?content=true` against a fixed allow-list of seven ATS
|
|
#: API hosts. The real run of 2026-09-23 hit it on the biggest employers —
|
|
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
|
|
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
|
|
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
|
|
#: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least
|
|
#: the titles land.
|
|
MAX_BYTES = 96 << 20
|
|
|
|
#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai`
|
|
#: failed a pull of 2026-09-28 with a read timeout. Measured live on
|
|
#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse
|
|
#: boards with the most open roles (databricks, stripe, anthropic, feverup,
|
|
#: datadog), with and without `pay_transparency=true`, cached and with a
|
|
#: cache-busting parameter (84 requests; CloudFront answered every one
|
|
#: "Miss", so all reached the origin): the body arrives in under 1 s once it
|
|
#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the
|
|
#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic,
|
|
#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise.
|
|
#: The flag made no difference that three runs could tell apart from that
|
|
#: noise. So a read may wait 30 s, about twice the worst first byte seen,
|
|
#: and a whole request, redirects and body included, 60 s (see `_get`),
|
|
#: about four times the slowest seen, which still leaves a 96 MB body room
|
|
#: on a link far slower than the one measured.
|
|
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
|
|
DEADLINE = 60.0
|
|
|
|
#: host -> (feed URL template, careers URL template). `{slug}` is filled in;
|
|
#: recruitee and personio put the slug in the hostname instead of the path,
|
|
#: and teamtailor's `{site}` is the whole host (see `_teamtailor_site`).
|
|
_BOARDS: dict[str, tuple[str, str]] = {
|
|
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs"
|
|
"?content=true&pay_transparency=true",
|
|
"https://job-boards.greenhouse.io/{slug}"),
|
|
"lever": ("https://api.lever.co/v0/postings/{slug}?mode=json",
|
|
"https://jobs.lever.co/{slug}"),
|
|
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}"
|
|
"?includeCompensation=true",
|
|
"https://jobs.ashbyhq.com/{slug}"),
|
|
"workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs",
|
|
"https://apply.workable.com/{slug}"),
|
|
"smartrecruiters": ("https://api.smartrecruiters.com/v1/companies/{slug}/postings",
|
|
"https://jobs.smartrecruiters.com/{slug}"),
|
|
"recruitee": ("https://{slug}.recruitee.com/api/offers/",
|
|
"https://{slug}.recruitee.com"),
|
|
"personio": ("https://{slug}.jobs.personio.de/xml",
|
|
"https://{slug}.jobs.personio.de"),
|
|
"teamtailor": ("https://{site}/jobs.rss", "https://{site}/jobs"),
|
|
}
|
|
|
|
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
|
#: the order a select offers them. What the Sources page's host select is
|
|
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
|
HOSTS: dict[str, str] = {
|
|
"greenhouse": "Greenhouse", "lever": "Lever", "ashby": "Ashby",
|
|
"workable": "Workable", "smartrecruiters": "SmartRecruiters",
|
|
"recruitee": "Recruitee", "personio": "Personio", "teamtailor": "Teamtailor",
|
|
}
|
|
assert set(HOSTS) == set(_BOARDS)
|
|
|
|
#: The hosts a request or a redirect may land on for a given board host — a
|
|
#: fixed API host for five of them, the company's own subdomain for the two
|
|
#: that put the slug in the hostname, and the one site a Teamtailor slug names.
|
|
#: Anything else is refused, at the
|
|
#: first hop and at every redirect: a feed is external content, and a board
|
|
#: row must never be able to point this fetch at an internal address.
|
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
|
fixed = {
|
|
"greenhouse": "boards-api.greenhouse.io",
|
|
"lever": "api.lever.co",
|
|
"ashby": "api.ashbyhq.com",
|
|
"workable": "apply.workable.com",
|
|
"smartrecruiters": "api.smartrecruiters.com",
|
|
}
|
|
if host in fixed:
|
|
want = fixed[host]
|
|
return lambda h: h == want
|
|
if host == "recruitee":
|
|
want = f"{slug}.recruitee.com"
|
|
return lambda h: h == want
|
|
if host == "personio":
|
|
want = f"{slug}.jobs.personio.de"
|
|
return lambda h: h == want
|
|
if host == "teamtailor":
|
|
want = _teamtailor_site(slug)
|
|
return lambda h: h == want
|
|
return lambda h: False
|
|
|
|
|
|
#: The host a Teamtailor career site is served from: `acme` is
|
|
#: acme.teamtailor.com, and a slug with a dot is a site on the company's own
|
|
#: domain, `careers.acme.com`, taken whole (the feed path is the same on both).
|
|
#: Only a DNS name with a letter in its last label is one, so an address
|
|
#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could be
|
|
#: pointed at; `_get` still refuses every host but this one, redirects included.
|
|
_TEAMTAILOR_HOST = re.compile(r"^[a-z0-9-]+(\.[a-z0-9-]+)*$")
|
|
|
|
|
|
def _teamtailor_site(slug: str) -> str:
|
|
slug = slug.strip().lower()
|
|
return slug if "." in slug else f"{slug}.teamtailor.com"
|
|
|
|
|
|
def _teamtailor_slug_ok(slug: str) -> bool:
|
|
site = _teamtailor_site(slug)
|
|
return bool(_TEAMTAILOR_HOST.match(site)) and not site.rsplit(".", 1)[-1].isdigit()
|
|
|
|
|
|
def _is_web_url(url: str) -> bool:
|
|
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
|
`javascript:` or `data:` link must never reach an href. Port of
|
|
openings.go's `webURL`."""
|
|
url = (url or "").strip()
|
|
if not url:
|
|
return False
|
|
parsed = httpx.URL(url)
|
|
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
|
|
|
|
|
class _Refused(Exception):
|
|
"""A board answered somewhere its allow-list does not cover."""
|
|
|
|
|
|
class _TooLarge(ValueError):
|
|
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
|
|
knows the old shape still catches it; `_fetch_rows` catches it by type to
|
|
retry Greenhouse without `?content=true`."""
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
|
method: str = "GET", payload: dict[str, Any] | None = None) -> bytes:
|
|
"""Fetch one board feed: https only, the host's own allow-list, at most
|
|
three redirect hops each checked the same way, and the body cut off at
|
|
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
|
sends `payload` as JSON, `{}` when there is none.
|
|
|
|
The body is streamed and counted as it arrives, so an oversized one is
|
|
refused at `MAX_BYTES` rather than after it has been read whole, and
|
|
the whole request, redirects included, must end within `DEADLINE`: the
|
|
per-read `TIMEOUTS` alone let a board that trickles bytes run on for
|
|
good. The deadline is checked between chunks, so it can be overrun by
|
|
at most one read timeout."""
|
|
allow = _allowed_host(host, slug)
|
|
deadline = time.monotonic() + DEADLINE
|
|
current = url
|
|
for _hop in range(4):
|
|
parsed = httpx.URL(current)
|
|
if parsed.scheme != "https" or not allow(parsed.host):
|
|
raise _Refused(f"{current} is not an allowed host for {host}")
|
|
with client.stream(
|
|
method, current, timeout=TIMEOUTS, follow_redirects=False,
|
|
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
|
|
json=(payload or {}) if method == "POST" else None,
|
|
) as response:
|
|
if response.is_redirect:
|
|
location = response.headers.get("location") or ""
|
|
if not location:
|
|
raise ValueError("redirect with no location")
|
|
current = str(httpx.URL(current).join(location))
|
|
continue
|
|
response.raise_for_status()
|
|
chunks: list[bytes] = []
|
|
size = 0
|
|
for chunk in response.iter_bytes():
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)")
|
|
if time.monotonic() > deadline:
|
|
raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s",
|
|
request=response.request)
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
raise ValueError("too many redirects")
|
|
|
|
|
|
def _parse_dt(value: Any) -> datetime | None:
|
|
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
|
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
|
|
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, int | float):
|
|
if value <= 0:
|
|
return None
|
|
seconds = value / 1000 if value > 10**12 else value
|
|
return datetime.fromtimestamp(seconds, tz=UTC)
|
|
text = str(value).strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
|
if not match:
|
|
try:
|
|
parsed = parsedate_to_datetime(text)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
# ------------------------------------------------------------- the parsers
|
|
|
|
#: What every parser turns a feed into: one dict per row, with these keys.
|
|
Row = dict[str, Any]
|
|
|
|
|
|
# ---------------------------------------------------------- structured pay
|
|
#
|
|
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
|
|
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
|
|
# `raw` into, per docs/decisions.md's "raw is transport, columns are
|
|
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
|
|
# structured pay on their list endpoint; Workable, SmartRecruiters and Teamtailor do not
|
|
# (checked live, 2026-09-29), so their parsers are untouched and `row.get`
|
|
# below reads a plain '' or None for them.
|
|
|
|
|
|
def _num(value: Any) -> float | None:
|
|
"""A positive number from whatever a board sends (int, float, or a
|
|
numeral string), or None for anything else, including zero and a
|
|
negative — a board's placeholder 0 (an unfilled field, not a real
|
|
figure) must not become a salary floor of zero."""
|
|
try:
|
|
number = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return number if number > 0 else None
|
|
|
|
|
|
def _yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
|
|
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
|
|
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
|
|
None, "")` when there is no currency, no bound, or `interval` is not one
|
|
of `factors`' own keys — which is how an hourly or a daily rate is left
|
|
out rather than annualised into a number nobody stated. `factors` maps
|
|
the board's own interval spelling to the multiplier that makes it
|
|
yearly: 1 for a figure already yearly, 12 for a monthly one."""
|
|
code = str(currency or "").strip().upper()
|
|
factor = factors.get(str(interval or "").strip())
|
|
if not code or not factor:
|
|
return None, None, ""
|
|
lo, hi = _num(min_value), _num(max_value)
|
|
if lo is None and hi is None:
|
|
return None, None, ""
|
|
return (int(lo * factor) if lo is not None else None,
|
|
int(hi * factor) if hi is not None else None, code)
|
|
|
|
|
|
def _greenhouse_salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
|
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
|
|
on the feed URL, see `_BOARDS`), which the list endpoint already carries
|
|
in the same request — no per-role fetch. Only when there is exactly one
|
|
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
|
|
one per office) gives no way to tell which office the role in hand is
|
|
at, and picking one would misrepresent the columns. Greenhouse's own
|
|
ranges are cents, and yearly by convention except an hourly or a daily
|
|
one, named as such in `title` with no separate field to check — those
|
|
are skipped rather than annualised into a number nobody stated."""
|
|
if len(ranges) != 1:
|
|
return None, None, ""
|
|
entry = ranges[0]
|
|
title = str(entry.get("title") or "").lower()
|
|
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
|
|
return None, None, ""
|
|
code = str(entry.get("currency_type") or "").strip().upper()
|
|
if not code:
|
|
return None, None, ""
|
|
lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents"))
|
|
if lo is None and hi is None:
|
|
return None, None, ""
|
|
return (int(lo / 100) if lo is not None else None,
|
|
int(hi / 100) if hi is not None else None, code)
|
|
|
|
|
|
def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
|
"""The yearly range from `compensation.summaryComponents` (needs
|
|
`?includeCompensation=true` on the feed URL, see `_BOARDS`): the first
|
|
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
|
|
an hourly one is left out rather than annualised."""
|
|
for component in components:
|
|
if component.get("compensationType") != "Salary":
|
|
continue
|
|
return _yearly_amount(component.get("minValue"), component.get("maxValue"),
|
|
component.get("currencyCode"), component.get("interval"),
|
|
{"1 YEAR": 1, "1 MONTH": 12})
|
|
return None, None, ""
|
|
|
|
|
|
# ------------------------------------------------------------- salary_text
|
|
#
|
|
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
|
|
# hourly and daily rates, a Greenhouse role with several pay zones, and a
|
|
# range with no currency are all dropped from those columns rather than
|
|
# guessed into a wrong yearly figure. `salary_text` is where that dropped
|
|
# information survives — one human-readable line per parser, written
|
|
# whether or not the pay conformed, so a person can read it and do their own
|
|
# sums and a stricter analysis can find the roles the columns left out
|
|
# (`Role.salary_state`, `app/models.py`).
|
|
|
|
#: Formats a number the way a salary line writes it: no decimals for a whole
|
|
#: figure (the normal case for a yearly or monthly one), two for anything
|
|
#: else (an hourly or daily rate quoted with cents).
|
|
def _fmt_amount(value: float) -> str:
|
|
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
|
|
|
|
|
|
def _bounds_words(lo: float | None, hi: float | None) -> str:
|
|
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
|
|
bound stated, or "" with neither."""
|
|
if lo is not None and hi is not None:
|
|
return _fmt_amount(lo) if lo == hi else f"{_fmt_amount(lo)} to {_fmt_amount(hi)}"
|
|
if lo is not None:
|
|
return f"from {_fmt_amount(lo)}"
|
|
if hi is not None:
|
|
return f"up to {_fmt_amount(hi)}"
|
|
return ""
|
|
|
|
|
|
def _pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
|
|
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
|
|
per year" — or "" when there is no bound to report at all. A currency
|
|
that was not stated does not drop the line, unlike the normalised
|
|
columns: it is said plainly instead, "80,000 to 100,000 per year (no
|
|
currency stated)", so the figure is not lost along with the currency."""
|
|
bounds = _bounds_words(lo, hi)
|
|
if not bounds:
|
|
return ""
|
|
tail = f" per {period}" if period else ""
|
|
code = str(currency or "").strip().upper()
|
|
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
|
|
|
|
|
def _greenhouse_salary_text(ranges: list[dict[str, Any]]) -> str:
|
|
"""One line per `pay_input_ranges` entry, joined with "; " — every range
|
|
Greenhouse states, not only the single one `_greenhouse_salary` accepts
|
|
for the columns. With more than one, each is labelled with the board's
|
|
own `title` ("Zone 1 Pay Range: ..."), since that is the only way to
|
|
tell the ranges apart; with just the one, the label adds nothing and is
|
|
left off. The hour/day check is the same one `_greenhouse_salary` uses
|
|
to keep those out of the yearly columns; here it only picks the word
|
|
that follows "per"."""
|
|
multi = len(ranges) > 1
|
|
lines = []
|
|
for entry in ranges:
|
|
title = str(entry.get("title") or "").strip()
|
|
low_title = title.lower()
|
|
period = ("hour" if any(w in low_title for w in ("hour", "/hr"))
|
|
else "day" if "day" in low_title else "year")
|
|
lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents"))
|
|
line = _pay_line(lo / 100 if lo is not None else None,
|
|
hi / 100 if hi is not None else None,
|
|
entry.get("currency_type"), period)
|
|
if not line:
|
|
continue
|
|
if multi:
|
|
label = title.rstrip(":").strip() or "Range"
|
|
line = f"{label}: {line}"
|
|
lines.append(line)
|
|
return "; ".join(lines)
|
|
|
|
|
|
def _lever_salary_text(salary_range: dict[str, Any]) -> str:
|
|
"""One line from `salaryRange`, whatever its `interval` — including an
|
|
hourly or daily one, which `_yearly_amount` drops from the columns.
|
|
Lever's interval is "per-<period>-<kind>" ("per-year-salary",
|
|
"per-hour-wage"); the middle word is the period."""
|
|
interval = str(salary_range.get("interval") or "")
|
|
parts = interval.split("-")
|
|
period = parts[1] if len(parts) >= 3 else ""
|
|
return _pay_line(_num(salary_range.get("min")), _num(salary_range.get("max")),
|
|
salary_range.get("currency"), period)
|
|
|
|
|
|
def _ashby_salary_text(compensation: dict[str, Any]) -> str:
|
|
"""One line per `summaryComponents` entry that states a bound, labelled
|
|
by the board's own `compensationType` when more than one does (a salary
|
|
range beside a bonus range, say); with just the one, the label is left
|
|
off, the same as `_greenhouse_salary_text`. `interval` is "<n>
|
|
<PERIOD>" ("1 YEAR", "1 HOUR"); the last word, lowered, is the period.
|
|
With nothing to build a line from, Ashby's own prose summary
|
|
(`compensationTierSummary`, else `scrapeableCompensationSalarySummary`)
|
|
is what is left to show."""
|
|
components = compensation.get("summaryComponents") or []
|
|
stated = [c for c in components if _num(c.get("minValue")) is not None
|
|
or _num(c.get("maxValue")) is not None]
|
|
lines = []
|
|
for component in stated:
|
|
period = str(component.get("interval") or "").split(" ")[-1].lower()
|
|
line = _pay_line(_num(component.get("minValue")), _num(component.get("maxValue")),
|
|
component.get("currencyCode"), period)
|
|
if not line:
|
|
continue
|
|
if len(stated) > 1:
|
|
label = str(component.get("compensationType") or "").strip()
|
|
line = f"{label}: {line}" if label else line
|
|
lines.append(line)
|
|
if lines:
|
|
return "; ".join(lines)
|
|
return str(compensation.get("compensationTierSummary")
|
|
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
|
|
|
|
|
def _recruitee_salary_text(salary: dict[str, Any]) -> str:
|
|
"""One line from Recruitee's `salary` object; `period` is already a
|
|
plain word ("year", "month")."""
|
|
period = str(salary.get("period") or "").strip().lower()
|
|
return _pay_line(_num(salary.get("min")), _num(salary.get("max")),
|
|
salary.get("currency"), period)
|
|
|
|
|
|
#: Personio's `salaryInformation/type` spellings, to the plain period word
|
|
#: `_pay_line` wants ("yearly" -> "year"). Anything else is passed through.
|
|
_PERSONIO_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
|
|
"daily": "day", "weekly": "week"}
|
|
|
|
|
|
def _personio_salary_text(salary_info: ET.Element) -> str:
|
|
"""One line from a `<salaryInformation>` element, whatever its `type`,
|
|
including an hourly or daily one that `_yearly_amount` drops from the
|
|
columns."""
|
|
period_raw = (salary_info.findtext("type") or "").strip().lower()
|
|
period = _PERSONIO_PERIODS.get(period_raw, period_raw)
|
|
return _pay_line(_num(salary_info.findtext("min")), _num(salary_info.findtext("max")),
|
|
salary_info.findtext("currencyCode"), period)
|
|
|
|
|
|
def _parse_greenhouse(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("jobs") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("absolute_url") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
departments = j.get("departments") or []
|
|
# Greenhouse ships the description HTML-entity-escaped inside the
|
|
# JSON string (`<p>`), so it needs unescaping before the tags
|
|
# themselves can be stripped.
|
|
content = html.unescape(j.get("content") or "")
|
|
ranges = j.get("pay_input_ranges") or []
|
|
salary_min, salary_max, currency = _greenhouse_salary(ranges)
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": ((j.get("location") or {}).get("name") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("first_published") or j.get("updated_at")),
|
|
"department": (departments[0].get("name") if departments else "") or "",
|
|
"description": strip_html(content),
|
|
"remote": None, # Greenhouse's feed does not say
|
|
"salary_text": _greenhouse_salary_text(ranges),
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_lever(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
if not isinstance(data, list):
|
|
return []
|
|
rows = []
|
|
for j in data:
|
|
title = (j.get("text") or "").strip()
|
|
url = (j.get("hostedUrl") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
categories = j.get("categories") or {}
|
|
description = j.get("description") or j.get("descriptionPlain") or ""
|
|
salary_range = j.get("salaryRange") or {}
|
|
salary_min, salary_max, currency = _yearly_amount(
|
|
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
|
|
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (categories.get("location") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("createdAt")),
|
|
"department": (categories.get("department") or categories.get("team") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": (j.get("workplaceType") or "").strip() or None,
|
|
"salary_text": _lever_salary_text(salary_range),
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_ashby(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("jobs") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
|
compensation = j.get("compensation") or {}
|
|
salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or [])
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (j.get("location") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("publishedAt")),
|
|
"department": (j.get("department") or j.get("team") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": j.get("isRemote"),
|
|
"salary_text": _ashby_salary_text(compensation),
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
|
return _workable_rows(json.loads(body), slug)
|
|
|
|
|
|
def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
|
|
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
|
|
Face's board answered on 2026-09-28. Ten results a page; the next page is
|
|
the same POST with `{"token": nextPage}`, and the last page has no
|
|
`nextPage` (see `_paged`). A result carries no URL and no text: the job page is
|
|
`https://apply.workable.com/{slug}/j/{shortcode}/`, and the description
|
|
is only on the per-job endpoint, one request per role, so `description`
|
|
stays empty here as it does for SmartRecruiters. `location` is an object
|
|
and `department` a list; `remote` is a flag beside `workplace`
|
|
(remote, hybrid, on_site)."""
|
|
rows = []
|
|
for j in data.get("results") or []:
|
|
title = (j.get("title") or "").strip()
|
|
shortcode = (j.get("shortcode") or "").strip()
|
|
if not title or not shortcode:
|
|
continue
|
|
loc = j.get("location") or {}
|
|
location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p)
|
|
dept = j.get("department") or ""
|
|
if isinstance(dept, list):
|
|
dept = ", ".join(d for d in dept if d)
|
|
rows.append({
|
|
"source_id": str(j.get("id") or shortcode),
|
|
"title": title,
|
|
"location": location,
|
|
"url": f"https://apply.workable.com/{slug}/j/{shortcode}/",
|
|
"posted": _parse_dt(j.get("published")),
|
|
"department": dept.strip(),
|
|
"description": "",
|
|
"remote": bool(j.get("remote")) or j.get("workplace") == "remote",
|
|
"salary_text": "", # Workable's list endpoint states no pay at all
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
|
|
return _smartrecruiters_rows(json.loads(body), slug)
|
|
|
|
|
|
def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
|
|
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
|
|
page, whatever `limit` asks for; the next is `?offset=` further on (see
|
|
`_paged`). The list omits the job description (only the
|
|
per-posting detail endpoint has it, and fetching one per row would be
|
|
hundreds of extra requests per board), so `description` stays empty here."""
|
|
rows = []
|
|
for j in data.get("content") or []:
|
|
title = (j.get("name") or "").strip()
|
|
job_id = str(j.get("id") or "")
|
|
if not title or not job_id:
|
|
continue
|
|
location = j.get("location") or {}
|
|
loc_text = (location.get("fullLocation")
|
|
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
|
rows.append({
|
|
"source_id": job_id,
|
|
"title": title,
|
|
"location": (loc_text or "").strip(),
|
|
"url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}",
|
|
"posted": _parse_dt(j.get("releasedDate")),
|
|
"department": ((j.get("department") or {}).get("label") or "").strip(),
|
|
"description": "",
|
|
"remote": location.get("remote"),
|
|
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_recruitee(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("offers") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("careers_url") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
|
salary_min, salary_max, currency = _yearly_amount(
|
|
salary.get("min"), salary.get("max"), salary.get("currency"),
|
|
salary.get("period"), {"year": 1, "month": 12})
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (j.get("location") or j.get("city") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("published_at") or j.get("created_at")),
|
|
"department": (j.get("department") or "").strip(),
|
|
"description": strip_html(j.get("description") or ""),
|
|
"remote": j.get("remote"),
|
|
"salary_text": _recruitee_salary_text(salary),
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_personio(body: bytes, slug: str) -> list[Row]:
|
|
"""`{slug}.jobs.personio.de/xml` -> `<workzag-jobs><position>...`. No job
|
|
URL in the feed itself; Personio's own page is `/job/{id}`, confirmed
|
|
live against a real posting (2026-09-23)."""
|
|
root = ET.fromstring(body)
|
|
rows = []
|
|
for position in root.findall("position"):
|
|
job_id = (position.findtext("id") or "").strip()
|
|
title = (position.findtext("name") or "").strip()
|
|
if not job_id or not title:
|
|
continue
|
|
description = " ".join(
|
|
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
|
salary_info = position.find("salaryInformation")
|
|
salary_min = salary_max = None
|
|
currency = ""
|
|
salary_text = ""
|
|
if salary_info is not None:
|
|
salary_min, salary_max, currency = _yearly_amount(
|
|
salary_info.findtext("min"), salary_info.findtext("max"),
|
|
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
|
|
{"yearly": 1, "monthly": 12})
|
|
salary_text = _personio_salary_text(salary_info)
|
|
rows.append({
|
|
"source_id": job_id,
|
|
"title": title,
|
|
"location": (position.findtext("office") or "").strip(),
|
|
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
|
|
"posted": _parse_dt(position.findtext("createdAt")),
|
|
"department": (position.findtext("department") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": "remote" in (position.findtext("office") or "").lower() or None,
|
|
"salary_text": salary_text,
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`.
|
|
_TT = "{https://teamtailor.com/locations}"
|
|
|
|
#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing
|
|
#: about where the role is worked and `temporary` is remote for a spell only,
|
|
#: so both stay unknown rather than become "onsite" or "remote".
|
|
_TEAMTAILOR_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"}
|
|
|
|
|
|
def _parse_teamtailor(body: bytes, slug: str) -> list[Row]:
|
|
return _teamtailor_rows(_teamtailor_items(body), slug)
|
|
|
|
|
|
def _teamtailor_items(body: bytes) -> list[ET.Element]:
|
|
"""`{site}/jobs.rss` -> `<rss><channel><item>...`, the feed every Teamtailor
|
|
career site serves (checked live on a dozen sites, 2026-09-29). A missing
|
|
`channel` is a page that is not a feed, and raises."""
|
|
channel = ET.fromstring(body).find("channel")
|
|
if channel is None:
|
|
raise ValueError("not a Teamtailor feed: no channel")
|
|
return channel.findall("item")
|
|
|
|
|
|
def _teamtailor_rows(items: list[ET.Element], slug: str) -> list[Row]:
|
|
"""One page of items. The item has the title, the job URL (on the site's
|
|
own domain), a `guid` uuid, `pubDate`, the HTML `description` in full,
|
|
`remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the
|
|
`tt:` namespace, `locations` (a list: `city` and `country` each, `name` free
|
|
text), `department` and `role`, either of which may be empty. `place` joins
|
|
the distinct locations with "; ", which `country_of` reads a location at a
|
|
time. No pay is stated anywhere in the feed."""
|
|
rows = []
|
|
for item in items:
|
|
title = (item.findtext("title") or "").strip()
|
|
url = (item.findtext("link") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
places: list[str] = []
|
|
for loc in item.findall(f"{_TT}locations/{_TT}location"):
|
|
city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip()
|
|
country = (loc.findtext(f"{_TT}country") or "").strip()
|
|
place = city if country == city else ", ".join(p for p in (city, country) if p)
|
|
if place and place not in places:
|
|
places.append(place)
|
|
rows.append({
|
|
"source_id": (item.findtext("guid") or url).strip(),
|
|
"title": title,
|
|
"location": "; ".join(places),
|
|
"url": url,
|
|
"posted": _parse_dt(item.findtext("pubDate")),
|
|
"department": ((item.findtext(f"{_TT}department") or "").strip()
|
|
or (item.findtext(f"{_TT}role") or "").strip()),
|
|
"description": html.unescape(strip_html(item.findtext("description") or "")),
|
|
"remote": _TEAMTAILOR_REMOTE.get((item.findtext("remoteStatus") or "").strip()),
|
|
"salary_text": "", # Teamtailor's feed states no pay at all
|
|
})
|
|
return rows
|
|
|
|
|
|
#: host -> parser. Every one takes the raw response body, plus the slug for
|
|
#: the hosts whose feed does not carry the URL or the company itself.
|
|
_PARSERS: dict[str, Callable[..., list[Row]]] = {
|
|
"greenhouse": lambda body, slug: _parse_greenhouse(body),
|
|
"lever": lambda body, slug: _parse_lever(body),
|
|
"ashby": lambda body, slug: _parse_ashby(body),
|
|
"workable": _parse_workable,
|
|
"smartrecruiters": _parse_smartrecruiters,
|
|
"recruitee": lambda body, slug: _parse_recruitee(body),
|
|
"personio": _parse_personio,
|
|
"teamtailor": _parse_teamtailor,
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------- the classifier
|
|
|
|
#: The fixed set of functions, and the keyword rules that map a title
|
|
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
|
|
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
|
|
#: twice: the first, narrower entry is checked before
|
|
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
|
#: matches the bare word "product", is checked only after all of those, so a
|
|
#: title like "Data Product Manager" keeps classifying as data/product by the
|
|
#: same rule order the Go tool used.
|
|
#:
|
|
#: Not a straight port any more: the Go tool matched substrings, which let
|
|
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
|
|
#: "Italia" — a European department like "Commercial Italy" filed as
|
|
#: engineering. `_classify_text` now matches on word boundaries. A handful of
|
|
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
|
|
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
|
|
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
|
|
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
|
|
#: meant to also catch "management", "engineering", "recruiting", and a
|
|
#: trailing boundary on those would stop them matching the plural and
|
|
#: gerund forms real postings use ("Software Developers", "Account
|
|
#: Managers").
|
|
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
|
"customer success/support", "recruiting/people", "finance/legal/ops")
|
|
|
|
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
("product", ("product manag", "product management", "product operations",
|
|
"program manag", "technical program")),
|
|
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
|
"people team", "people operations", "human resources",
|
|
"hr", "hrbp")),
|
|
("design", ("design", "ux", "user experience", "creative")),
|
|
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
|
"seo", "social media", "demand gen", "events")),
|
|
("customer success/support", ("customer success", "customer support",
|
|
"customer experience", "customer service",
|
|
"support", "success", "implementation",
|
|
"onboarding", "professional services",
|
|
"solutions consult", "technical account",
|
|
"customer engineer", "deployment")),
|
|
("sales", ("sales", "account executive", "account manager", "account director",
|
|
"business development", "bdr", "sdr", "revenue", "partnership",
|
|
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
|
"pre-sales", "presales", "channel",
|
|
# "Commercial" is what French, Italian and Spanish org charts
|
|
# call the sales function ("Commercial Italy" is a sales
|
|
# department; see the docstring above), and the rest are the
|
|
# sales words of the seed's other languages that "sales"
|
|
# itself does not already cover (German, Dutch, Danish,
|
|
# Norwegian, Swedish, French) — a modest set, not a full port;
|
|
# `extract` is the rest.
|
|
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
|
|
"försäljning", "vente")),
|
|
("data", ("data", "analytics", "machine learning", "ml",
|
|
"applied science", "research scientist", "statistic")),
|
|
("product", ("product manag", "product management", "program manag",
|
|
"product operations", "technical program", "product")),
|
|
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
|
|
"compliance", "operations", "ops", "administrat",
|
|
"office", "procurement", "strategy", "corporate",
|
|
"facilities", "risk", "audit", "tax", "treasury",
|
|
"payroll", "executive assistant", "chief of staff",
|
|
"workplace", "supply chain", "logistics", "fraud",
|
|
"trust & safety", "trust and safety", "policy")),
|
|
("engineering", ("engineer", "software", "developer", "infrastructure",
|
|
"platform", "devops", "sre", "site reliability", "security",
|
|
"information technology", "it", "technology",
|
|
"technical staff", "hardware", "qa", "quality assurance",
|
|
"architect", "research", "science", "scientist", "systems",
|
|
"network", "cloud", "mobile", "frontend", "backend",
|
|
"full stack", "fullstack", "web",
|
|
# The developer/engineer words of the seed's other
|
|
# languages, same reasoning as sales' above.
|
|
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
|
|
"utvikler", "utvecklare", "ingenjör", "ingeniør",
|
|
"développeur", "developpeur", "ingénieur")),
|
|
)
|
|
|
|
#: Keywords that must match a whole word, not just its start: each one is
|
|
#: itself the first letters of an unrelated word that would otherwise false-
|
|
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
|
|
#: not listed here keeps the old prefix behaviour on purpose — see the
|
|
#: `_RULES` docstring above.
|
|
_WHOLE_WORD = frozenset({
|
|
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
|
|
"sre", "bdr", "sdr", "gtm",
|
|
})
|
|
|
|
|
|
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
|
|
trailing = r"\b" if keyword in _WHOLE_WORD else ""
|
|
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
|
|
|
|
|
|
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
|
|
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
|
|
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
|
|
for function, keywords in _RULES)
|
|
|
|
|
|
def _classify_text(text: str) -> str:
|
|
stripped = (text or "").strip()
|
|
if not stripped:
|
|
return ""
|
|
for function, patterns in _COMPILED_RULES:
|
|
if any(pattern.search(stripped) for pattern in patterns):
|
|
return function
|
|
return ""
|
|
|
|
|
|
def classify(department: str, title: str) -> str:
|
|
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
|
|
when neither names one — `''` and not `"other"`, which would claim a
|
|
knowledge the classifier does not have (`extract` may still fill it from
|
|
reading the posting; see `Store.set_extracted`).
|
|
|
|
Title before department: a department is often a generic umbrella
|
|
("Operations", "Engineering & Product") that says less about the role
|
|
than its title does. A hand-checked sample of 60 roles where the two
|
|
orders disagreed (docs/decisions.md) found title-first right about twice
|
|
as often as department-first, mostly because a specific title beats a
|
|
broad department name, so title now wins; department is the fallback
|
|
when the title matches nothing."""
|
|
return _classify_text(title or "") or _classify_text(department or "") or ""
|
|
|
|
|
|
def is_known_target(target: str) -> bool:
|
|
"""`host:slug`, with `host` one of `_BOARDS` and a non-empty slug (for
|
|
Teamtailor one that names a host, see `_teamtailor_slug_ok`). What
|
|
the target validator refuses: `app companies add` and the pull job
|
|
both call this before touching the network."""
|
|
host, sep, slug = target.partition(":")
|
|
return (bool(sep) and host in _BOARDS and bool(slug.strip())
|
|
and (host != "teamtailor" or _teamtailor_slug_ok(slug)))
|
|
|
|
|
|
def careers_url(target: str) -> str:
|
|
"""The company's own careers page for `host:slug`, or "" for an unknown
|
|
host. What `app companies add` stores alongside the board."""
|
|
host, _, slug = target.partition(":")
|
|
if host not in _BOARDS:
|
|
return ""
|
|
return _BOARDS[host][1].format(slug=slug, site=_teamtailor_site(slug))
|
|
|
|
|
|
#: Pages read from one paged board at most, and the pause between two of
|
|
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
|
|
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause keeps Workable, which answers
|
|
#: a burst with 429, from seeing one.
|
|
MAX_PAGES = 100
|
|
PAGE_DELAY = 0.5
|
|
|
|
#: SmartRecruiters' page size, which is also the most it serves per page.
|
|
SMARTRECRUITERS_PAGE = 100
|
|
|
|
#: Teamtailor's page size, asked for as `per_page`. A feed with neither
|
|
#: parameter answers with every role (73 of 73 at Shine, 2026-09-29), but says
|
|
#: nowhere that it is the whole list, so this reads by page to an empty one;
|
|
#: `per_page=200` was answered in full too, and the cursor advances by what a
|
|
#: page returned, so a smaller cap on the host's side would not skip a role.
|
|
TEAMTAILOR_PAGE = 50
|
|
|
|
#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its
|
|
#: raw results, the total it states (None for a host that states none), and the
|
|
#: cursor of the next page (None after the last). The first call gets cursor
|
|
#: None.
|
|
Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any]
|
|
|
|
|
|
def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
|
url = _BOARDS["workable"][0].format(slug=slug)
|
|
data = json.loads(_get(client, url, "workable", slug, method="POST",
|
|
payload={"token": cursor} if cursor else {}))
|
|
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
|
|
|
|
|
|
def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
|
offset = int(cursor or 0)
|
|
url = (_BOARDS["smartrecruiters"][0].format(slug=slug)
|
|
+ f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}")
|
|
data = json.loads(_get(client, url, "smartrecruiters", slug))
|
|
content = data.get("content") or []
|
|
return data, content, data.get("totalFound"), offset + len(content) if content else None
|
|
|
|
|
|
def _teamtailor_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
|
offset = int(cursor or 0)
|
|
url = (_BOARDS["teamtailor"][0].format(site=_teamtailor_site(slug))
|
|
+ f"?per_page={TEAMTAILOR_PAGE}&offset={offset}")
|
|
items = _teamtailor_items(_get(client, url, "teamtailor", slug))
|
|
results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items]
|
|
return items, results, None, offset + len(items) if items else None
|
|
|
|
|
|
#: host -> (one page, that page's rows) for the hosts that page.
|
|
_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page],
|
|
Callable[[dict[str, Any], str], list[Row]]]] = {
|
|
"workable": (_workable_page, _workable_rows),
|
|
"smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows),
|
|
"teamtailor": (_teamtailor_page, _teamtailor_rows),
|
|
}
|
|
|
|
|
|
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|
"""Every page of a Workable, SmartRecruiters or Teamtailor board,
|
|
`PAGE_DELAY` apart, until the host has no next page, the stated total is
|
|
reached, or `MAX_PAGES`. Returns the rows and whether the board is
|
|
complete: as many distinct results read as the host's stated total, or,
|
|
for a host that states none (Teamtailor), the read ended on an empty page
|
|
(a page with nothing new in it ends the read short of that, incomplete,
|
|
so an offset the host ignores cannot loop). Counted on the raw
|
|
results, so a row the parser drops does not make a board incomplete, and
|
|
on distinct ids, so a page that shifted under an offset and repeated a
|
|
role does not make it complete.
|
|
|
|
A 429 after the first page stops the read with what it has, incomplete;
|
|
one on the first page raises like any other failure."""
|
|
host, _, slug = target.partition(":")
|
|
page_of, rows_of = _PAGED[host]
|
|
rows: dict[str, Row] = {}
|
|
seen: set[str] = set()
|
|
total: Any = None
|
|
cursor: Any = None
|
|
for page in range(MAX_PAGES):
|
|
if page:
|
|
time.sleep(PAGE_DELAY)
|
|
try:
|
|
data, results, total, cursor = page_of(client, slug, cursor)
|
|
except httpx.HTTPStatusError as exc:
|
|
if not page or exc.response.status_code != 429:
|
|
raise
|
|
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
|
|
target, page + 1, len(seen), total)
|
|
return list(rows.values()), False
|
|
before = len(seen)
|
|
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
|
for row in rows_of(data, slug):
|
|
rows.setdefault(row["source_id"], row)
|
|
if (cursor is None or (isinstance(total, int) and len(seen) >= total)
|
|
or (total is None and len(seen) == before)):
|
|
break
|
|
complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None
|
|
if not complete:
|
|
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen),
|
|
"an unstated total" if total is None else total)
|
|
return list(rows.values()), complete
|
|
|
|
|
|
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|
"""One board, fetched and parsed: its rows and whether they are the whole
|
|
board. Raises on any failure — refused host, timeout, a feed shape none
|
|
of the parsers understand. A host that does not page answers with the
|
|
whole board, so it is complete whenever it parses; a paged one is
|
|
complete when `_paged` says so.
|
|
|
|
A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES`
|
|
gets one retry without it: the description is lost, but the titles land
|
|
rather than the whole board being skipped."""
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
host, _, slug = target.partition(":")
|
|
if host in _PAGED:
|
|
return _paged(client, target)
|
|
url = _BOARDS[host][0].format(slug=slug)
|
|
try:
|
|
body = _get(client, url, host, slug, method="GET")
|
|
except _TooLarge:
|
|
if host != "greenhouse" or "?" not in url:
|
|
raise
|
|
log.warning("ats: %s: body exceeded %d bytes with content=true; "
|
|
"retrying without it", target, MAX_BYTES)
|
|
url = url.split("?", 1)[0]
|
|
body = _get(client, url, host, slug, method="GET")
|
|
return _PARSERS[host](body, slug), True
|
|
|
|
|
|
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
|
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
|
"does this board answer, what does it list, and is that all of it". For
|
|
`app companies add|check`, which ask that question about one board at a
|
|
time outside any pull. Raises the same way `_fetch_rows` does."""
|
|
with httpx.Client() as client:
|
|
return _fetch_rows(client, target)
|
|
|
|
|
|
def failure_reason(exc: BaseException) -> str:
|
|
"""One short line for why a board did not answer: the status of an HTTP
|
|
error, else the exception's class and the first line of its message."""
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
|
lines = str(exc).strip().splitlines()
|
|
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
|
return text[:200]
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class ATSSource(Source):
|
|
name = "ats"
|
|
description = "Company job boards: Greenhouse, Lever, Ashby, Workable, " \
|
|
"SmartRecruiters, Recruitee, Personio, Teamtailor."
|
|
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
|
#: company}` — `url` carries the `host:slug` target and `label` the
|
|
#: company name, which is how `Source.targets()`'s generic dict reading
|
|
#: hands both back without a plugin-specific override.
|
|
subscription_field = "boards"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="ATS boards",
|
|
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# Handed in by the batch that runs this plugin, as the marker of
|
|
# the pull run that queued it (see `app/jobs/pull.py`), and never
|
|
# read from state, where a later run may have replaced it. A
|
|
# plugin run on its own — a test, a dry run — has no run to stamp
|
|
# itself against, so it makes one up.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
|
|
for target, label in self.targets():
|
|
host, _, slug = target.partition(":")
|
|
failing = f"ats:failing:{target}"
|
|
try:
|
|
rows, complete = _fetch_rows(client, target)
|
|
except Exception as exc:
|
|
log.warning("ats: %s failed (%s)", target, exc)
|
|
# The first failure in a row sets `since`, later ones only
|
|
# the reason; `app.jobs.pull` closes the board's roles
|
|
# once `since` is `STALE_DAYS` old.
|
|
try:
|
|
since = json.loads(self.state.get(failing, "") or "{}").get("since")
|
|
except ValueError:
|
|
since = None
|
|
self.state[failing] = json.dumps({
|
|
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
|
"reason": failure_reason(exc)})
|
|
continue
|
|
self.state.pop(failing, None)
|
|
|
|
seen_ids = []
|
|
for row in rows:
|
|
# The compound id, matching `roles.source_id` — what
|
|
# `Store.mark_closed` compares against, not the board's own
|
|
# bare job id.
|
|
compound_id = f"{host}:{slug}:{row['source_id']}"
|
|
seen_ids.append(compound_id)
|
|
yield Item(
|
|
source_id=compound_id,
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=(row["description"] or "")[:300],
|
|
body=row["description"],
|
|
published_at=row["posted"],
|
|
subscription=label,
|
|
kind="role",
|
|
raw={
|
|
"company": label,
|
|
"place": row["location"],
|
|
"department": row["department"],
|
|
"function": classify(row["department"], row["title"]),
|
|
"remote": row["remote"],
|
|
# Every parser sets salary_text, empty or not (see
|
|
# "salary_text" above); only Ashby, Greenhouse,
|
|
# Lever, Recruitee and Personio's also set the
|
|
# normalised figures (see "structured pay"
|
|
# above), so the rest leave them at row.get's
|
|
# default of no figure.
|
|
"salary_text": row.get("salary_text", ""),
|
|
"salary_min": row.get("salary_min"),
|
|
"salary_max": row.get("salary_max"),
|
|
"currency": row.get("currency", ""),
|
|
"board": host,
|
|
"slug": slug,
|
|
},
|
|
)
|
|
# Stamped only once the board has parsed without raising, so a
|
|
# board that blew up half way through is neither marked ok nor
|
|
# has a partial id list treated as the whole of it. `complete`
|
|
# says whether the ids are the whole board, which the pull
|
|
# needs before it closes what they leave out.
|
|
self.state[f"ats:ok:{target}"] = run_marker
|
|
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
|
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|