From b02b72b4aae2c5d3ebec7aadc757449aa2e13af8 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:03:37 +0200 Subject: [PATCH] Move the eight hosts into a module each under plugins/atshosts and build the ats plugin's tables from them Co-Authored-By: Claude Opus 5.5 --- plugins/ats.py | 902 ++++------------------------ plugins/atshosts/__init__.py | 113 ++++ plugins/atshosts/_shared.py | 188 ++++++ plugins/atshosts/ashby.py | 121 ++++ plugins/atshosts/greenhouse.py | 138 +++++ plugins/atshosts/lever.py | 120 ++++ plugins/atshosts/personio.py | 100 +++ plugins/atshosts/recruitee.py | 83 +++ plugins/atshosts/smartrecruiters.py | 89 +++ plugins/atshosts/teamtailor.py | 136 +++++ plugins/atshosts/workable.py | 84 +++ tests/test_ats.py | 11 +- 12 files changed, 1296 insertions(+), 789 deletions(-) create mode 100644 plugins/atshosts/__init__.py create mode 100644 plugins/atshosts/_shared.py create mode 100644 plugins/atshosts/ashby.py create mode 100644 plugins/atshosts/greenhouse.py create mode 100644 plugins/atshosts/lever.py create mode 100644 plugins/atshosts/personio.py create mode 100644 plugins/atshosts/recruitee.py create mode 100644 plugins/atshosts/smartrecruiters.py create mode 100644 plugins/atshosts/teamtailor.py create mode 100644 plugins/atshosts/workable.py diff --git a/plugins/ats.py b/plugins/ats.py index ba6b618..bc18502 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -1,69 +1,69 @@ -"""ATS boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, -Teamtailor. +"""ATS boards: a company's own careers page on one of the hosts in +`plugins/atshosts/`, one module per host. -Every one of these hosts is a company's own careers page speaking public, -unauthenticated JSON (Personio, Teamtailor: XML) — first-party, no scraping, and a role -that drops off the board is *closed*, which is the cheapest staleness check -there is. +Every one of these hosts serves a company's own list of roles as public, +unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that +drops off the board is *closed*, which is the cheapest staleness check there +is. What a host says and how its feed reads is its module's; this file holds +what every host shares: the fetch and its allow-list, paging and +completeness, the classifier, and the plugin itself. -The field map and the redirect allow-list are ported from -`openings.go` and `cmd/seedgen/main.go` in +The allow-list and the classifier are ported from `openings.go` and +`cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including -`classify`'s nine functions and their keywords. Workable, SmartRecruiters, -Recruitee and Personio are new: jobhunt-hq only read Greenhouse, Lever and -Ashby, so those four are written from one real board of each host, fetched -while this file was written (2026-09-23; Workable on 2026-09-28, when the -first board with roles was seen), rather than from the Go source. Teamtailor -is the same, written from live feeds on 2026-09-29. - -The body cap is not a straight port — see `MAX_BYTES` below. +`classify`'s nine functions and their keywords. The body cap is not a +straight port — see `MAX_BYTES` below. A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, -`lever:acme`, `teamtailor:acme`. Every board is fetched independently; one that fails or times +`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what +its slug is. Every board is fetched independently; one that fails or times out is logged, recorded as failing under `ats:failing:` (see `ATSSource.fetch`), and skipped, and the rest continue. -A Teamtailor slug is the `acme` of acme.teamtailor.com, or, for a career site -on the company's own domain, that whole host name: `teamtailor:careers.acme.com` -(see `_teamtailor_site`). - -Workable and SmartRecruiters page their list and state a total; Teamtailor -pages by offset and states none; every other host answers with the whole board -at once. A board with a total counts as complete only when it read as many -distinct roles as the total says, one without when it read up to an empty page -(see `_paged`). +A host that pages states a total or does not. A board with a total counts as +complete only when it read as many distinct roles as the total says, one +without when it read up to an empty page (see `_paged`); a host that answers +with the whole board at once is complete whenever it parses. """ from __future__ import annotations -import html import json import logging import re import time -import xml.etree.ElementTree as ET from collections.abc import Callable, Iterator from datetime import UTC, datetime -from email.utils import parsedate_to_datetime from typing import Any import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, - SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html, + SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, ) +from plugins import atshosts +from plugins.atshosts._shared import Get, Row + +# The names the parsers had here before they moved to their host modules, +# kept for the callers and the tests that still reach for them on `ats`. +from plugins.atshosts._shared import ( # noqa: F401 + Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount, +) +from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401 +from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401 + log = logging.getLogger(__name__) #: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every -#: URL fetched here is `?content=true` against a fixed allow-list of seven ATS -#: API hosts. The real run of 2026-09-23 hit it on the biggest employers — +#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's +#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers — #: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and #: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised #: to 96 MB so those boards fit; a board that still exceeds it gets one retry -#: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least -#: the titles land. +#: at its host's `fallback_url` where it has one (Greenhouse, without +#: `?content=true`, see `_fetch_rows`), so at least the titles land. MAX_BYTES = 96 << 20 #: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai` @@ -84,96 +84,20 @@ MAX_BYTES = 96 << 20 TIMEOUTS = httpx.Timeout(10.0, read=30.0) DEADLINE = 60.0 -#: host -> (feed URL template, careers URL template). `{slug}` is filled in; -#: recruitee and personio put the slug in the hostname instead of the path, -#: and teamtailor's `{site}` is the whole host (see `_teamtailor_site`). -_BOARDS: dict[str, tuple[str, str]] = { - "greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs" - "?content=true&pay_transparency=true", - "https://job-boards.greenhouse.io/{slug}"), - "lever": ("https://api.lever.co/v0/postings/{slug}?mode=json", - "https://jobs.lever.co/{slug}"), - "ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}" - "?includeCompensation=true", - "https://jobs.ashbyhq.com/{slug}"), - "workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs", - "https://apply.workable.com/{slug}"), - "smartrecruiters": ("https://api.smartrecruiters.com/v1/companies/{slug}/postings", - "https://jobs.smartrecruiters.com/{slug}"), - "recruitee": ("https://{slug}.recruitee.com/api/offers/", - "https://{slug}.recruitee.com"), - "personio": ("https://{slug}.jobs.personio.de/xml", - "https://{slug}.jobs.personio.de"), - "teamtailor": ("https://{site}/jobs.rss", "https://{site}/jobs"), -} - #: host -> the name the host gives itself: the hosts this plugin reads, in #: the order a select offers them. What the Sources page's host select is #: rendered from, so it cannot offer a host `is_known_target` refuses. -HOSTS: dict[str, str] = { - "greenhouse": "Greenhouse", "lever": "Lever", "ashby": "Ashby", - "workable": "Workable", "smartrecruiters": "SmartRecruiters", - "recruitee": "Recruitee", "personio": "Personio", "teamtailor": "Teamtailor", -} -assert set(HOSTS) == set(_BOARDS) +HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()} + -#: The hosts a request or a redirect may land on for a given board host — a -#: fixed API host for five of them, the company's own subdomain for the two -#: that put the slug in the hostname, and the one site a Teamtailor slug names. -#: Anything else is refused, at the -#: first hop and at every redirect: a feed is external content, and a board -#: row must never be able to point this fetch at an internal address. def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: - fixed = { - "greenhouse": "boards-api.greenhouse.io", - "lever": "api.lever.co", - "ashby": "api.ashbyhq.com", - "workable": "apply.workable.com", - "smartrecruiters": "api.smartrecruiters.com", - } - if host in fixed: - want = fixed[host] - return lambda h: h == want - if host == "recruitee": - want = f"{slug}.recruitee.com" - return lambda h: h == want - if host == "personio": - want = f"{slug}.jobs.personio.de" - return lambda h: h == want - if host == "teamtailor": - want = _teamtailor_site(slug) - return lambda h: h == want - return lambda h: False - - -#: The host a Teamtailor career site is served from: `acme` is -#: acme.teamtailor.com, and a slug with a dot is a site on the company's own -#: domain, `careers.acme.com`, taken whole (the feed path is the same on both). -#: Only a DNS name with a letter in its last label is one, so an address -#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could be -#: pointed at; `_get` still refuses every host but this one, redirects included. -_TEAMTAILOR_HOST = re.compile(r"^[a-z0-9-]+(\.[a-z0-9-]+)*$") - - -def _teamtailor_site(slug: str) -> str: - slug = slug.strip().lower() - return slug if "." in slug else f"{slug}.teamtailor.com" - - -def _teamtailor_slug_ok(slug: str) -> bool: - site = _teamtailor_site(slug) - return bool(_TEAMTAILOR_HOST.match(site)) and not site.rsplit(".", 1)[-1].isdigit() - - -def _is_web_url(url: str) -> bool: - """A trimmed http(s) URL, or refuse — a feed is external content and a - `javascript:` or `data:` link must never reach an href. Port of - openings.go's `webURL`.""" - url = (url or "").strip() - if not url: - return False - parsed = httpx.URL(url) - return parsed.scheme in ("http", "https") and bool(parsed.host) + """The hosts a request or a redirect may land on for one board: what its + host module's `allowed` says, which is one fixed API host or the + company's own site. Anything else is refused, at the first hop and at + every redirect: a feed is external content, and a board row must never + be able to point this fetch at an internal address.""" + module = atshosts.HOSTS.get(host) + return module.allowed(slug) if module is not None else lambda h: False class _Refused(Exception): @@ -183,15 +107,18 @@ class _Refused(Exception): class _TooLarge(ValueError): """A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only knows the old shape still catches it; `_fetch_rows` catches it by type to - retry Greenhouse without `?content=true`.""" + retry at the host's `fallback_url`.""" def _get(client: httpx.Client, url: str, host: str, slug: str, *, - method: str = "GET", payload: dict[str, Any] | None = None) -> bytes: - """Fetch one board feed: https only, the host's own allow-list, at most - three redirect hops each checked the same way, and the body cut off at - `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST - sends `payload` as JSON, `{}` when there is none. + method: str = "GET", payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None) -> bytes: + """Fetch one URL for a board: https only, the host's own allow-list, at + most three redirect hops each checked the same way, and the body cut off + at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST + sends `payload` as JSON, `{}` when there is none. `headers` are sent on + top of the User-Agent and Accept every request carries, and replace + either where they name it. The body is streamed and counted as it arrives, so an oversized one is refused at `MAX_BYTES` rather than after it has been read whole, and @@ -200,6 +127,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, good. The deadline is checked between chunks, so it can be overrun by at most one read timeout.""" allow = _allowed_host(host, slug) + sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {}) deadline = time.monotonic() + DEADLINE current = url for _hop in range(4): @@ -207,8 +135,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, if parsed.scheme != "https" or not allow(parsed.host): raise _Refused(f"{current} is not an allowed host for {host}") with client.stream( - method, current, timeout=TIMEOUTS, follow_redirects=False, - headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"}, + method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent, json=(payload or {}) if method == "POST" else None, ) as response: if response.is_redirect: @@ -232,567 +159,27 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, raise ValueError("too many redirects") -def _parse_dt(value: Any) -> datetime | None: - """An aware datetime from whatever a board sends: ISO-8601 with an - offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC", - or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now".""" - if value is None: - return None - if isinstance(value, int | float): - if value <= 0: - return None - seconds = value / 1000 if value > 10**12 else value - return datetime.fromtimestamp(seconds, tz=UTC) - text = str(value).strip() - if not text: - return None - try: - parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) - except ValueError: - match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text) - if not match: - try: - parsed = parsedate_to_datetime(text) - except (TypeError, ValueError): - return None - return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) - parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00") - return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) +def _getter(client: httpx.Client, host: str, slug: str) -> Get: + """The `get` a host module is handed: `_get` bound to one board. `_get` + is looked up when a request is made, not when this is built, so a test + that replaces `ats._get` reaches every host; `payload` and `headers` are + passed on only when given, which keeps a replacement that predates them + working.""" + def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None) -> bytes: + extra: dict[str, Any] = {} + if payload is not None: + extra["payload"] = payload + if headers: + extra["headers"] = headers + return _get(client, url, host, slug, method=method, **extra) + return get -# ------------------------------------------------------------- the parsers - -#: What every parser turns a feed into: one dict per row, with these keys. -Row = dict[str, Any] - - -# ---------------------------------------------------------- structured pay -# -# A board's own salary range, mapped to yearly `salary_min`/`salary_max` -# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks -# `raw` into, per docs/decisions.md's "raw is transport, columns are -# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give -# structured pay on their list endpoint; Workable, SmartRecruiters and Teamtailor do not -# (checked live, 2026-09-29), so their parsers are untouched and `row.get` -# below reads a plain '' or None for them. - - -def _num(value: Any) -> float | None: - """A positive number from whatever a board sends (int, float, or a - numeral string), or None for anything else, including zero and a - negative — a board's placeholder 0 (an unfilled field, not a real - figure) must not become a salary floor of zero.""" - try: - number = float(value) - except (TypeError, ValueError): - return None - return number if number > 0 else None - - -def _yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any, - factors: dict[str, int]) -> tuple[int | None, int | None, str]: - """`(salary_min, salary_max, currency)` as yearly integers, or `(None, - None, "")` when there is no currency, no bound, or `interval` is not one - of `factors`' own keys — which is how an hourly or a daily rate is left - out rather than annualised into a number nobody stated. `factors` maps - the board's own interval spelling to the multiplier that makes it - yearly: 1 for a figure already yearly, 12 for a monthly one.""" - code = str(currency or "").strip().upper() - factor = factors.get(str(interval or "").strip()) - if not code or not factor: - return None, None, "" - lo, hi = _num(min_value), _num(max_value) - if lo is None and hi is None: - return None, None, "" - return (int(lo * factor) if lo is not None else None, - int(hi * factor) if hi is not None else None, code) - - -def _greenhouse_salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: - """The yearly range from `pay_input_ranges` (needs `pay_transparency=true` - on the feed URL, see `_BOARDS`), which the list endpoint already carries - in the same request — no per-role fetch. Only when there is exactly one - range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations", - one per office) gives no way to tell which office the role in hand is - at, and picking one would misrepresent the columns. Greenhouse's own - ranges are cents, and yearly by convention except an hourly or a daily - one, named as such in `title` with no separate field to check — those - are skipped rather than annualised into a number nobody stated.""" - if len(ranges) != 1: - return None, None, "" - entry = ranges[0] - title = str(entry.get("title") or "").lower() - if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")): - return None, None, "" - code = str(entry.get("currency_type") or "").strip().upper() - if not code: - return None, None, "" - lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents")) - if lo is None and hi is None: - return None, None, "" - return (int(lo / 100) if lo is not None else None, - int(hi / 100) if hi is not None else None, code) - - -def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: - """The yearly range from `compensation.summaryComponents` (needs - `?includeCompensation=true` on the feed URL, see `_BOARDS`): the first - `Salary` component, yearly or monthly (a monthly one multiplied by 12); - an hourly one is left out rather than annualised.""" - for component in components: - if component.get("compensationType") != "Salary": - continue - return _yearly_amount(component.get("minValue"), component.get("maxValue"), - component.get("currencyCode"), component.get("interval"), - {"1 YEAR": 1, "1 MONTH": 12}) - return None, None, "" - - -# ------------------------------------------------------------- salary_text -# -# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose: -# hourly and daily rates, a Greenhouse role with several pay zones, and a -# range with no currency are all dropped from those columns rather than -# guessed into a wrong yearly figure. `salary_text` is where that dropped -# information survives — one human-readable line per parser, written -# whether or not the pay conformed, so a person can read it and do their own -# sums and a stricter analysis can find the roles the columns left out -# (`Role.salary_state`, `app/models.py`). - -#: Formats a number the way a salary line writes it: no decimals for a whole -#: figure (the normal case for a yearly or monthly one), two for anything -#: else (an hourly or daily rate quoted with cents). -def _fmt_amount(value: float) -> str: - return f"{int(value):,}" if value == int(value) else f"{value:,.2f}" - - -def _bounds_words(lo: float | None, hi: float | None) -> str: - """"70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one - bound stated, or "" with neither.""" - if lo is not None and hi is not None: - return _fmt_amount(lo) if lo == hi else f"{_fmt_amount(lo)} to {_fmt_amount(hi)}" - if lo is not None: - return f"from {_fmt_amount(lo)}" - if hi is not None: - return f"up to {_fmt_amount(hi)}" - return "" - - -def _pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str: - """One statement of pay as a human-readable line — "EUR 70,000 to 90,000 - per year" — or "" when there is no bound to report at all. A currency - that was not stated does not drop the line, unlike the normalised - columns: it is said plainly instead, "80,000 to 100,000 per year (no - currency stated)", so the figure is not lost along with the currency.""" - bounds = _bounds_words(lo, hi) - if not bounds: - return "" - tail = f" per {period}" if period else "" - code = str(currency or "").strip().upper() - return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)" - - -def _greenhouse_salary_text(ranges: list[dict[str, Any]]) -> str: - """One line per `pay_input_ranges` entry, joined with "; " — every range - Greenhouse states, not only the single one `_greenhouse_salary` accepts - for the columns. With more than one, each is labelled with the board's - own `title` ("Zone 1 Pay Range: ..."), since that is the only way to - tell the ranges apart; with just the one, the label adds nothing and is - left off. The hour/day check is the same one `_greenhouse_salary` uses - to keep those out of the yearly columns; here it only picks the word - that follows "per".""" - multi = len(ranges) > 1 - lines = [] - for entry in ranges: - title = str(entry.get("title") or "").strip() - low_title = title.lower() - period = ("hour" if any(w in low_title for w in ("hour", "/hr")) - else "day" if "day" in low_title else "year") - lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents")) - line = _pay_line(lo / 100 if lo is not None else None, - hi / 100 if hi is not None else None, - entry.get("currency_type"), period) - if not line: - continue - if multi: - label = title.rstrip(":").strip() or "Range" - line = f"{label}: {line}" - lines.append(line) - return "; ".join(lines) - - -def _lever_salary_text(salary_range: dict[str, Any]) -> str: - """One line from `salaryRange`, whatever its `interval` — including an - hourly or daily one, which `_yearly_amount` drops from the columns. - Lever's interval is "per--" ("per-year-salary", - "per-hour-wage"); the middle word is the period.""" - interval = str(salary_range.get("interval") or "") - parts = interval.split("-") - period = parts[1] if len(parts) >= 3 else "" - return _pay_line(_num(salary_range.get("min")), _num(salary_range.get("max")), - salary_range.get("currency"), period) - - -def _ashby_salary_text(compensation: dict[str, Any]) -> str: - """One line per `summaryComponents` entry that states a bound, labelled - by the board's own `compensationType` when more than one does (a salary - range beside a bonus range, say); with just the one, the label is left - off, the same as `_greenhouse_salary_text`. `interval` is " - " ("1 YEAR", "1 HOUR"); the last word, lowered, is the period. - With nothing to build a line from, Ashby's own prose summary - (`compensationTierSummary`, else `scrapeableCompensationSalarySummary`) - is what is left to show.""" - components = compensation.get("summaryComponents") or [] - stated = [c for c in components if _num(c.get("minValue")) is not None - or _num(c.get("maxValue")) is not None] - lines = [] - for component in stated: - period = str(component.get("interval") or "").split(" ")[-1].lower() - line = _pay_line(_num(component.get("minValue")), _num(component.get("maxValue")), - component.get("currencyCode"), period) - if not line: - continue - if len(stated) > 1: - label = str(component.get("compensationType") or "").strip() - line = f"{label}: {line}" if label else line - lines.append(line) - if lines: - return "; ".join(lines) - return str(compensation.get("compensationTierSummary") - or compensation.get("scrapeableCompensationSalarySummary") or "") - - -def _recruitee_salary_text(salary: dict[str, Any]) -> str: - """One line from Recruitee's `salary` object; `period` is already a - plain word ("year", "month").""" - period = str(salary.get("period") or "").strip().lower() - return _pay_line(_num(salary.get("min")), _num(salary.get("max")), - salary.get("currency"), period) - - -#: Personio's `salaryInformation/type` spellings, to the plain period word -#: `_pay_line` wants ("yearly" -> "year"). Anything else is passed through. -_PERSONIO_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour", - "daily": "day", "weekly": "week"} - - -def _personio_salary_text(salary_info: ET.Element) -> str: - """One line from a `` element, whatever its `type`, - including an hourly or daily one that `_yearly_amount` drops from the - columns.""" - period_raw = (salary_info.findtext("type") or "").strip().lower() - period = _PERSONIO_PERIODS.get(period_raw, period_raw) - return _pay_line(_num(salary_info.findtext("min")), _num(salary_info.findtext("max")), - salary_info.findtext("currencyCode"), period) - - -def _parse_greenhouse(body: bytes) -> list[Row]: - data = json.loads(body) - rows = [] - for j in data.get("jobs") or []: - title = (j.get("title") or "").strip() - url = (j.get("absolute_url") or "").strip() - if not title or not _is_web_url(url): - continue - departments = j.get("departments") or [] - # Greenhouse ships the description HTML-entity-escaped inside the - # JSON string (`<p>`), so it needs unescaping before the tags - # themselves can be stripped. - content = html.unescape(j.get("content") or "") - ranges = j.get("pay_input_ranges") or [] - salary_min, salary_max, currency = _greenhouse_salary(ranges) - rows.append({ - "source_id": str(j.get("id") or ""), - "title": title, - "location": ((j.get("location") or {}).get("name") or "").strip(), - "url": url, - "posted": _parse_dt(j.get("first_published") or j.get("updated_at")), - "department": (departments[0].get("name") if departments else "") or "", - "description": strip_html(content), - "remote": None, # Greenhouse's feed does not say - "salary_text": _greenhouse_salary_text(ranges), - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -def _parse_lever(body: bytes) -> list[Row]: - data = json.loads(body) - if not isinstance(data, list): - return [] - rows = [] - for j in data: - title = (j.get("text") or "").strip() - url = (j.get("hostedUrl") or "").strip() - if not title or not _is_web_url(url): - continue - categories = j.get("categories") or {} - description = j.get("description") or j.get("descriptionPlain") or "" - salary_range = j.get("salaryRange") or {} - salary_min, salary_max, currency = _yearly_amount( - salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"), - salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12}) - rows.append({ - "source_id": str(j.get("id") or ""), - "title": title, - "location": (categories.get("location") or "").strip(), - "url": url, - "posted": _parse_dt(j.get("createdAt")), - "department": (categories.get("department") or categories.get("team") or "").strip(), - "description": strip_html(description), - "remote": (j.get("workplaceType") or "").strip() or None, - "salary_text": _lever_salary_text(salary_range), - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -def _parse_ashby(body: bytes) -> list[Row]: - data = json.loads(body) - rows = [] - for j in data.get("jobs") or []: - title = (j.get("title") or "").strip() - url = (j.get("jobUrl") or j.get("applyUrl") or "").strip() - if not title or not _is_web_url(url): - continue - description = j.get("descriptionHtml") or j.get("descriptionPlain") or "" - compensation = j.get("compensation") or {} - salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or []) - rows.append({ - "source_id": str(j.get("id") or ""), - "title": title, - "location": (j.get("location") or "").strip(), - "url": url, - "posted": _parse_dt(j.get("publishedAt")), - "department": (j.get("department") or j.get("team") or "").strip(), - "description": strip_html(description), - "remote": j.get("isRemote"), - "salary_text": _ashby_salary_text(compensation), - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -def _parse_workable(body: bytes, slug: str) -> list[Row]: - return _workable_rows(json.loads(body), slug) - - -def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]: - """One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs - -> `{"total": int, "results": [...], "nextPage": token}`, as Hugging - Face's board answered on 2026-09-28. Ten results a page; the next page is - the same POST with `{"token": nextPage}`, and the last page has no - `nextPage` (see `_paged`). A result carries no URL and no text: the job page is - `https://apply.workable.com/{slug}/j/{shortcode}/`, and the description - is only on the per-job endpoint, one request per role, so `description` - stays empty here as it does for SmartRecruiters. `location` is an object - and `department` a list; `remote` is a flag beside `workplace` - (remote, hybrid, on_site).""" - rows = [] - for j in data.get("results") or []: - title = (j.get("title") or "").strip() - shortcode = (j.get("shortcode") or "").strip() - if not title or not shortcode: - continue - loc = j.get("location") or {} - location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p) - dept = j.get("department") or "" - if isinstance(dept, list): - dept = ", ".join(d for d in dept if d) - rows.append({ - "source_id": str(j.get("id") or shortcode), - "title": title, - "location": location, - "url": f"https://apply.workable.com/{slug}/j/{shortcode}/", - "posted": _parse_dt(j.get("published")), - "department": dept.strip(), - "description": "", - "remote": bool(j.get("remote")) or j.get("workplace") == "remote", - "salary_text": "", # Workable's list endpoint states no pay at all - }) - return rows - - -def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]: - return _smartrecruiters_rows(json.loads(body), slug) - - -def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]: - """One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings - -> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a - page, whatever `limit` asks for; the next is `?offset=` further on (see - `_paged`). The list omits the job description (only the - per-posting detail endpoint has it, and fetching one per row would be - hundreds of extra requests per board), so `description` stays empty here.""" - rows = [] - for j in data.get("content") or []: - title = (j.get("name") or "").strip() - job_id = str(j.get("id") or "") - if not title or not job_id: - continue - location = j.get("location") or {} - loc_text = (location.get("fullLocation") - or ", ".join(p for p in (location.get("city"), location.get("country")) if p)) - rows.append({ - "source_id": job_id, - "title": title, - "location": (loc_text or "").strip(), - "url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}", - "posted": _parse_dt(j.get("releasedDate")), - "department": ((j.get("department") or {}).get("label") or "").strip(), - "description": "", - "remote": location.get("remote"), - "salary_text": "", # SmartRecruiters' list endpoint states no pay at all - }) - return rows - - -def _parse_recruitee(body: bytes) -> list[Row]: - data = json.loads(body) - rows = [] - for j in data.get("offers") or []: - title = (j.get("title") or "").strip() - url = (j.get("careers_url") or "").strip() - if not title or not _is_web_url(url): - continue - salary = j.get("salary") if isinstance(j.get("salary"), dict) else {} - salary_min, salary_max, currency = _yearly_amount( - salary.get("min"), salary.get("max"), salary.get("currency"), - salary.get("period"), {"year": 1, "month": 12}) - rows.append({ - "source_id": str(j.get("id") or ""), - "title": title, - "location": (j.get("location") or j.get("city") or "").strip(), - "url": url, - "posted": _parse_dt(j.get("published_at") or j.get("created_at")), - "department": (j.get("department") or "").strip(), - "description": strip_html(j.get("description") or ""), - "remote": j.get("remote"), - "salary_text": _recruitee_salary_text(salary), - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -def _parse_personio(body: bytes, slug: str) -> list[Row]: - """`{slug}.jobs.personio.de/xml` -> `...`. No job - URL in the feed itself; Personio's own page is `/job/{id}`, confirmed - live against a real posting (2026-09-23).""" - root = ET.fromstring(body) - rows = [] - for position in root.findall("position"): - job_id = (position.findtext("id") or "").strip() - title = (position.findtext("name") or "").strip() - if not job_id or not title: - continue - description = " ".join( - (value.text or "") for value in position.findall("jobDescriptions/jobDescription/value")) - salary_info = position.find("salaryInformation") - salary_min = salary_max = None - currency = "" - salary_text = "" - if salary_info is not None: - salary_min, salary_max, currency = _yearly_amount( - salary_info.findtext("min"), salary_info.findtext("max"), - salary_info.findtext("currencyCode"), salary_info.findtext("type"), - {"yearly": 1, "monthly": 12}) - salary_text = _personio_salary_text(salary_info) - rows.append({ - "source_id": job_id, - "title": title, - "location": (position.findtext("office") or "").strip(), - "url": f"https://{slug}.jobs.personio.de/job/{job_id}", - "posted": _parse_dt(position.findtext("createdAt")), - "department": (position.findtext("department") or "").strip(), - "description": strip_html(description), - "remote": "remote" in (position.findtext("office") or "").lower() or None, - "salary_text": salary_text, - "salary_min": salary_min, - "salary_max": salary_max, - "currency": currency, - }) - return rows - - -#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`. -_TT = "{https://teamtailor.com/locations}" - -#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing -#: about where the role is worked and `temporary` is remote for a spell only, -#: so both stay unknown rather than become "onsite" or "remote". -_TEAMTAILOR_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"} - - -def _parse_teamtailor(body: bytes, slug: str) -> list[Row]: - return _teamtailor_rows(_teamtailor_items(body), slug) - - -def _teamtailor_items(body: bytes) -> list[ET.Element]: - """`{site}/jobs.rss` -> `...`, the feed every Teamtailor - career site serves (checked live on a dozen sites, 2026-09-29). A missing - `channel` is a page that is not a feed, and raises.""" - channel = ET.fromstring(body).find("channel") - if channel is None: - raise ValueError("not a Teamtailor feed: no channel") - return channel.findall("item") - - -def _teamtailor_rows(items: list[ET.Element], slug: str) -> list[Row]: - """One page of items. The item has the title, the job URL (on the site's - own domain), a `guid` uuid, `pubDate`, the HTML `description` in full, - `remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the - `tt:` namespace, `locations` (a list: `city` and `country` each, `name` free - text), `department` and `role`, either of which may be empty. `place` joins - the distinct locations with "; ", which `country_of` reads a location at a - time. No pay is stated anywhere in the feed.""" - rows = [] - for item in items: - title = (item.findtext("title") or "").strip() - url = (item.findtext("link") or "").strip() - if not title or not _is_web_url(url): - continue - places: list[str] = [] - for loc in item.findall(f"{_TT}locations/{_TT}location"): - city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip() - country = (loc.findtext(f"{_TT}country") or "").strip() - place = city if country == city else ", ".join(p for p in (city, country) if p) - if place and place not in places: - places.append(place) - rows.append({ - "source_id": (item.findtext("guid") or url).strip(), - "title": title, - "location": "; ".join(places), - "url": url, - "posted": _parse_dt(item.findtext("pubDate")), - "department": ((item.findtext(f"{_TT}department") or "").strip() - or (item.findtext(f"{_TT}role") or "").strip()), - "description": html.unescape(strip_html(item.findtext("description") or "")), - "remote": _TEAMTAILOR_REMOTE.get((item.findtext("remoteStatus") or "").strip()), - "salary_text": "", # Teamtailor's feed states no pay at all - }) - return rows - - -#: host -> parser. Every one takes the raw response body, plus the slug for -#: the hosts whose feed does not carry the URL or the company itself. -_PARSERS: dict[str, Callable[..., list[Row]]] = { - "greenhouse": lambda body, slug: _parse_greenhouse(body), - "lever": lambda body, slug: _parse_lever(body), - "ashby": lambda body, slug: _parse_ashby(body), - "workable": _parse_workable, - "smartrecruiters": _parse_smartrecruiters, - "recruitee": lambda body, slug: _parse_recruitee(body), - "personio": _parse_personio, - "teamtailor": _parse_teamtailor, -} +#: host -> parser, for the hosts whose module parses a whole body. Every one +#: takes the raw response body and the slug. +_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { + name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")} # --------------------------------------------------------- the classifier @@ -923,98 +310,46 @@ def classify(department: str, title: str) -> str: def is_known_target(target: str) -> bool: - """`host:slug`, with `host` one of `_BOARDS` and a non-empty slug (for - Teamtailor one that names a host, see `_teamtailor_slug_ok`). What - the target validator refuses: `app companies add` and the pull job - both call this before touching the network.""" + """`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's + `slug_ok` accepts. What the target validator refuses: `app companies add` + and the pull job both call this before touching the network.""" host, sep, slug = target.partition(":") - return (bool(sep) and host in _BOARDS and bool(slug.strip()) - and (host != "teamtailor" or _teamtailor_slug_ok(slug))) + module = atshosts.HOSTS.get(host) + return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug) def careers_url(target: str) -> str: """The company's own careers page for `host:slug`, or "" for an unknown host. What `app companies add` stores alongside the board.""" host, _, slug = target.partition(":") - if host not in _BOARDS: - return "" - return _BOARDS[host][1].format(slug=slug, site=_teamtailor_site(slug)) + module = atshosts.HOSTS.get(host) + return module.careers_url(slug) if module is not None else "" #: Pages read from one paged board at most, and the pause between two of #: them. Workable answers ten roles a page, SmartRecruiters a hundred and -#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause keeps Workable, which answers -#: a burst with 429, from seeing one. +#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause +#: keeps Workable, which answers a burst with 429, from seeing one. MAX_PAGES = 100 PAGE_DELAY = 0.5 -#: SmartRecruiters' page size, which is also the most it serves per page. -SMARTRECRUITERS_PAGE = 100 - -#: Teamtailor's page size, asked for as `per_page`. A feed with neither -#: parameter answers with every role (73 of 73 at Shine, 2026-09-29), but says -#: nowhere that it is the whole list, so this reads by page to an empty one; -#: `per_page=200` was answered in full too, and the cursor advances by what a -#: page returned, so a smaller cap on the host's side would not skip a role. -TEAMTAILOR_PAGE = 50 - -#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its -#: raw results, the total it states (None for a host that states none), and the -#: cursor of the next page (None after the last). The first call gets cursor -#: None. -Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any] - - -def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page: - url = _BOARDS["workable"][0].format(slug=slug) - data = json.loads(_get(client, url, "workable", slug, method="POST", - payload={"token": cursor} if cursor else {})) - return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None - - -def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page: - offset = int(cursor or 0) - url = (_BOARDS["smartrecruiters"][0].format(slug=slug) - + f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}") - data = json.loads(_get(client, url, "smartrecruiters", slug)) - content = data.get("content") or [] - return data, content, data.get("totalFound"), offset + len(content) if content else None - - -def _teamtailor_page(client: httpx.Client, slug: str, cursor: Any) -> Page: - offset = int(cursor or 0) - url = (_BOARDS["teamtailor"][0].format(site=_teamtailor_site(slug)) - + f"?per_page={TEAMTAILOR_PAGE}&offset={offset}") - items = _teamtailor_items(_get(client, url, "teamtailor", slug)) - results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items] - return items, results, None, offset + len(items) if items else None - - -#: host -> (one page, that page's rows) for the hosts that page. -_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page], - Callable[[dict[str, Any], str], list[Row]]]] = { - "workable": (_workable_page, _workable_rows), - "smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows), - "teamtailor": (_teamtailor_page, _teamtailor_rows), -} - def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: - """Every page of a Workable, SmartRecruiters or Teamtailor board, - `PAGE_DELAY` apart, until the host has no next page, the stated total is - reached, or `MAX_PAGES`. Returns the rows and whether the board is - complete: as many distinct results read as the host's stated total, or, - for a host that states none (Teamtailor), the read ended on an empty page - (a page with nothing new in it ends the read short of that, incomplete, - so an offset the host ignores cannot loop). Counted on the raw - results, so a row the parser drops does not make a board incomplete, and - on distinct ids, so a page that shifted under an offset and repeated a - role does not make it complete. + """Every page of a board whose host pages, `PAGE_DELAY` apart, until the + host has no next page, the stated total is reached, or `MAX_PAGES`. + Returns the rows and whether the board is complete: as many distinct + results read as the host's stated total, or, for a host that states none + (Teamtailor), the read ended on an empty page (a page with nothing new in + it ends the read short of that, incomplete, so an offset the host ignores + cannot loop). Counted on the raw results, so a row the parser drops does + not make a board incomplete, and on distinct ids, so a page that shifted + under an offset and repeated a role does not make it complete. A 429 after the first page stops the read with what it has, incomplete; one on the first page raises like any other failure.""" host, _, slug = target.partition(":") - page_of, rows_of = _PAGED[host] + module = atshosts.HOSTS[host] + get = _getter(client, host, slug) rows: dict[str, Row] = {} seen: set[str] = set() total: Any = None @@ -1023,7 +358,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: if page: time.sleep(PAGE_DELAY) try: - data, results, total, cursor = page_of(client, slug, cursor) + data, results, total, cursor = module.page(get, slug, cursor) except httpx.HTTPStatusError as exc: if not page or exc.response.status_code != 429: raise @@ -1032,7 +367,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: return list(rows.values()), False before = len(seen) seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results} - for row in rows_of(data, slug): + for row in module.rows(data, slug): rows.setdefault(row["source_id"], row) if (cursor is None or (isinstance(total, int) and len(seen) >= total) or (total is None and len(seen) == before)): @@ -1045,31 +380,32 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: - """One board, fetched and parsed: its rows and whether they are the whole - board. Raises on any failure — refused host, timeout, a feed shape none - of the parsers understand. A host that does not page answers with the - whole board, so it is complete whenever it parses; a paged one is - complete when `_paged` says so. + """One board's listing, fetched and parsed: its rows and whether they are + the whole board. Raises on any failure — refused host, timeout, a feed + shape the host's parser does not understand. A host that does not page + answers with the whole board, so it is complete whenever it parses; a + paged one is complete when `_paged` says so. - A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES` - gets one retry without it: the description is lost, but the titles land - rather than the whole board being skipped.""" + A body that exceeds `MAX_BYTES` gets one retry at the host's + `fallback_url` where it has one (Greenhouse, without `?content=true`): + the description is lost, but the titles land rather than the whole board + being skipped.""" if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") host, _, slug = target.partition(":") - if host in _PAGED: + module = atshosts.HOSTS[host] + if hasattr(module, "page"): return _paged(client, target) - url = _BOARDS[host][0].format(slug=slug) + get = _getter(client, host, slug) try: - body = _get(client, url, host, slug, method="GET") + body = get(module.feed_url(slug)) except _TooLarge: - if host != "greenhouse" or "?" not in url: + if not hasattr(module, "fallback_url"): raise - log.warning("ats: %s: body exceeded %d bytes with content=true; " - "retrying without it", target, MAX_BYTES) - url = url.split("?", 1)[0] - body = _get(client, url, host, slug, method="GET") - return _PARSERS[host](body, slug), True + log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed", + target, MAX_BYTES) + body = get(module.fallback_url(slug)) + return module.parse(body, slug), True def fetch_board(target: str) -> tuple[list[Row], bool]: @@ -1095,8 +431,7 @@ def failure_reason(exc: BaseException) -> str: class ATSSource(Source): name = "ats" - description = "Company job boards: Greenhouse, Lever, Ashby, Workable, " \ - "SmartRecruiters, Recruitee, Personio, Teamtailor." + description = f"Company job boards: {', '.join(HOSTS.values())}." #: The raw entries under `sources.ats.boards` are `{"url": target, "label": #: company}` — `url` carries the `host:slug` target and `label` the #: company name, which is how `Source.targets()`'s generic dict reading @@ -1169,11 +504,10 @@ class ATSSource(Source): "function": classify(row["department"], row["title"]), "remote": row["remote"], # Every parser sets salary_text, empty or not (see - # "salary_text" above); only Ashby, Greenhouse, - # Lever, Recruitee and Personio's also set the - # normalised figures (see "structured pay" - # above), so the rest leave them at row.get's - # default of no figure. + # "salary_text" in `atshosts/_shared.py`); only + # the hosts that state structured pay also set the + # normalised figures, so the rest leave them at + # row.get's default of no figure. "salary_text": row.get("salary_text", ""), "salary_min": row.get("salary_min"), "salary_max": row.get("salary_max"), diff --git a/plugins/atshosts/__init__.py b/plugins/atshosts/__init__.py new file mode 100644 index 0000000..4e17355 --- /dev/null +++ b/plugins/atshosts/__init__.py @@ -0,0 +1,113 @@ +"""The hosts the `ats` plugin reads, one module each. + +A host registers by existing: every module in this package whose name does +not start with `_` is loaded, in order of its name, and checked against the +contract below when the package is imported, so a module that breaks it fails +the import with a message naming the module and what it lacks, rather than a +pull that fails board by board. `plugins/ats.py` builds its allow-list, +feeds and pages from `HOSTS`, and `app/jobs/discover.py` its patterns and +probes. The package sits beside `ats.py` rather than being `plugins/ats/`, +since `desk.sources.registry` loads `plugins/*.py` by path and the tests +patch names on the `ats` module. + +A module is named after its host and holds: + + NAME = "lever" # the host of a target `lever:`, and the module's name + LABEL = "Lever" # what the Sources page's select shows + PATTERNS = (r"...",) # discovery: over a URL; the slug is group 1, or the + # non-empty groups joined by "/" when there are more + + def slug_ok(slug) -> bool # strict; a slug that fails is no target + def allowed(slug) -> Callable[[str], bool] # the host names a fetch may reach + def careers_url(slug) -> str # the page a person opens + +and one of two ways to read a board: + + def feed_url(slug) -> str # the whole board in one answer, + def parse(body: bytes, slug) -> list[Row] # read with `parse` + + def page(get: Get, slug, cursor) -> Page # a paged board, one page a call, + def rows(decoded, slug) -> list[Row] # its rows from `page`'s decoded page + +and, optionally: + + def fallback_url(slug) -> str # a smaller feed, read once when + # `feed_url`'s answer is over the byte cap + def robots_page(slug) -> str # the URL the site's robots.txt must allow + def detail(get: Get, slug, row: Row) -> Row # the row filled from the role's own page + def probe_url(slug) -> str # discovery: answers 2xx only for a board + def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough + def probe_slugs(guess) -> tuple[str, ...] # the slugs a guessed name is probed as + +`Get`, `Row` and `Page` are in `_shared`, with the helpers several hosts use. +A host module imports nothing from `plugins.ats`: the fetch it is handed is +`ats._get` bound to the board, so every request a host makes passes the same +allow-list, redirect rule, byte cap and deadline. +""" + +from __future__ import annotations + +import importlib +import pkgutil +import re +from collections.abc import Iterable +from types import ModuleType + +#: What every host module defines. +REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url") + +#: The functions a module may define; each must be callable if present. +FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows", + "fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs") + + +def problems(module: ModuleType, name: str | None = None) -> list[str]: + """What `module` lacks or gets wrong against the contract, one phrase + each; empty for a module that keeps it. `name` is the module's own name + in the package, which `NAME` must equal.""" + out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)] + if name is not None and getattr(module, "NAME", name) != name: + out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}") + if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL): + out.append("LABEL is not a non-empty string") + patterns = getattr(module, "PATTERNS", ()) + if isinstance(patterns, str) or not isinstance(patterns, Iterable): + out.append("PATTERNS is not a tuple of strings") + else: + for pattern in patterns: + try: + re.compile(pattern) + except (TypeError, re.error) as exc: + out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})") + out += [f"{attr} is not callable" for attr in FUNCTIONS + if hasattr(module, attr) and not callable(getattr(module, attr))] + if hasattr(module, "page"): + if not hasattr(module, "rows"): + out.append("page without rows") + elif not (hasattr(module, "feed_url") and hasattr(module, "parse")): + out.append("neither feed_url and parse nor page and rows") + if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"): + out.append("probe_ok without probe_url") + if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"): + out.append("probe_slugs without probe_url") + return out + + +def load(path: Iterable[str], package: str) -> dict[str, ModuleType]: + """`NAME -> module` for every module of `package` (found on `path`) whose + name does not start with `_`, sorted by name. Raises ImportError naming + the first module that breaks the contract.""" + hosts: dict[str, ModuleType] = {} + for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name): + if info.name.startswith("_"): + continue + module = importlib.import_module(f"{package}.{info.name}") + if found := problems(module, info.name): + raise ImportError(f"{package}.{info.name} breaks the host contract: " + + "; ".join(found)) + hosts[module.NAME] = module + return hosts + + +#: host -> its module, in order of name. +HOSTS: dict[str, ModuleType] = load(__path__, __name__) diff --git a/plugins/atshosts/_shared.py b/plugins/atshosts/_shared.py new file mode 100644 index 0000000..dfdd79e --- /dev/null +++ b/plugins/atshosts/_shared.py @@ -0,0 +1,188 @@ +"""What the host modules share: the row and page shapes, the fetch they are +handed, and the date, URL and salary helpers more than one host needs. + +Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both +import it without either importing the other. The leading underscore keeps +`plugins.atshosts` from listing it as a host. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime +from typing import Any, Protocol + +import httpx + +#: What every parser turns a feed into: one dict per row, with the keys +#: `source_id`, `title`, `location`, `url`, `posted`, `department`, +#: `description`, `remote` and `salary_text`, and, where a host states pay, +#: `salary_min`, `salary_max` and `currency`. +Row = dict[str, Any] + +#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded +#: page, its raw results (dicts with an `id`, counted for completeness), the +#: total it states (None for a host that states none), and the cursor of the +#: next page (None after the last). The first call gets cursor None. +Page = tuple[Any, list[dict[str, Any]], Any, Any] + + +class Get(Protocol): + """The fetch a host is handed: `ats.py`'s own, bound to one board, so the + allow-list, the redirect rule, the byte cap and the deadline apply to + every request a host makes. A POST sends `payload` as JSON; `headers` are + added to (and may replace) the default User-Agent and Accept. Returns the + body, and raises on a non-2xx answer as `httpx` does.""" + + def __call__(self, url: str, *, method: str = "GET", + payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None) -> bytes: ... + + +def is_web_url(url: str) -> bool: + """A trimmed http(s) URL, or refuse — a feed is external content and a + `javascript:` or `data:` link must never reach an href. Port of + openings.go's `webURL`.""" + url = (url or "").strip() + if not url: + return False + parsed = httpx.URL(url) + return parsed.scheme in ("http", "https") and bool(parsed.host) + + +def only(host: str) -> Callable[[str], bool]: + """The allow-list of a board that is fetched from one host name only, + which is every host so far.""" + return lambda h: h == host + + +#: A slug that is one URL path segment: what the hosts that put the slug in +#: the path accept, and nothing that could add a path, a query or a fragment. +_SEGMENT = re.compile(r"^[A-Za-z0-9._~%-]+$") + +#: A slug that is one DNS label, for the hosts that put it in the host name. +_LABEL = re.compile(r"^[A-Za-z0-9-]+$") + + +def path_slug_ok(slug: str) -> bool: + return bool(_SEGMENT.match(slug)) and slug not in (".", "..") + + +def label_slug_ok(slug: str) -> bool: + return bool(_LABEL.match(slug)) + + +def parse_dt(value: Any) -> datetime | None: + """An aware datetime from whatever a board sends: ISO-8601 with an + offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC", + or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now".""" + if value is None: + return None + if isinstance(value, int | float): + if value <= 0: + return None + seconds = value / 1000 if value > 10**12 else value + return datetime.fromtimestamp(seconds, tz=UTC) + text = str(value).strip() + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text) + if not match: + try: + parsed = parsedate_to_datetime(text) + except (TypeError, ValueError): + return None + return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) + parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00") + return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) + + +# ---------------------------------------------------------- structured pay +# +# A board's own salary range, mapped to yearly `salary_min`/`salary_max` +# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks +# `raw` into, per docs/decisions.md's "raw is transport, columns are +# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give +# structured pay on their list endpoint; Workable, SmartRecruiters and +# Teamtailor do not (checked live, 2026-09-29), so their rows leave the +# columns out and `ATSSource.fetch` reads a plain '' or None for them. + + +def num(value: Any) -> float | None: + """A positive number from whatever a board sends (int, float, or a + numeral string), or None for anything else, including zero and a + negative — a board's placeholder 0 (an unfilled field, not a real + figure) must not become a salary floor of zero.""" + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if number > 0 else None + + +def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any, + factors: dict[str, int]) -> tuple[int | None, int | None, str]: + """`(salary_min, salary_max, currency)` as yearly integers, or `(None, + None, "")` when there is no currency, no bound, or `interval` is not one + of `factors`' own keys — which is how an hourly or a daily rate is left + out rather than annualised into a number nobody stated. `factors` maps + the board's own interval spelling to the multiplier that makes it + yearly: 1 for a figure already yearly, 12 for a monthly one.""" + code = str(currency or "").strip().upper() + factor = factors.get(str(interval or "").strip()) + if not code or not factor: + return None, None, "" + lo, hi = num(min_value), num(max_value) + if lo is None and hi is None: + return None, None, "" + return (int(lo * factor) if lo is not None else None, + int(hi * factor) if hi is not None else None, code) + + +# ------------------------------------------------------------- salary_text +# +# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose: +# hourly and daily rates, a Greenhouse role with several pay zones, and a +# range with no currency are all dropped from those columns rather than +# guessed into a wrong yearly figure. `salary_text` is where that dropped +# information survives — one human-readable line per parser, written +# whether or not the pay conformed, so a person can read it and do their own +# sums and a stricter analysis can find the roles the columns left out +# (`Role.salary_state`, `app/models.py`). + +#: Formats a number the way a salary line writes it: no decimals for a whole +#: figure (the normal case for a yearly or monthly one), two for anything +#: else (an hourly or daily rate quoted with cents). +def fmt_amount(value: float) -> str: + return f"{int(value):,}" if value == int(value) else f"{value:,.2f}" + + +def bounds_words(lo: float | None, hi: float | None) -> str: + """"70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one + bound stated, or "" with neither.""" + if lo is not None and hi is not None: + return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}" + if lo is not None: + return f"from {fmt_amount(lo)}" + if hi is not None: + return f"up to {fmt_amount(hi)}" + return "" + + +def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str: + """One statement of pay as a human-readable line — "EUR 70,000 to 90,000 + per year" — or "" when there is no bound to report at all. A currency + that was not stated does not drop the line, unlike the normalised + columns: it is said plainly instead, "80,000 to 100,000 per year (no + currency stated)", so the figure is not lost along with the currency.""" + bounds = bounds_words(lo, hi) + if not bounds: + return "" + tail = f" per {period}" if period else "" + code = str(currency or "").strip().upper() + return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)" diff --git a/plugins/atshosts/ashby.py b/plugins/atshosts/ashby.py new file mode 100644 index 0000000..8685c38 --- /dev/null +++ b/plugins/atshosts/ashby.py @@ -0,0 +1,121 @@ +"""Ashby: `ashby:`, the whole board in one JSON answer. A +board name may carry a dot (`jimdo.com`) or be a uuid. + +The field map is ported from `openings.go` in +[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT). +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount + +NAME = "ashby" +LABEL = "Ashby" +PATTERNS = ( + r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)", + r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)", +) + +_API = "api.ashbyhq.com" + + +def slug_ok(slug: str) -> bool: + return path_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_API) + + +def careers_url(slug: str) -> str: + return f"https://jobs.ashbyhq.com/{slug}" + + +def feed_url(slug: str) -> str: + return f"https://{_API}/posting-api/job-board/{slug}?includeCompensation=true" + + +def probe_url(slug: str) -> str: + return f"https://{_API}/posting-api/job-board/{slug}" + + +def probe_ok(body: bytes) -> bool: + """Ashby answers an empty listing for an unknown name, so a listing + without jobs counts as no board.""" + return b'"jobs":[' in body and b'"jobs":[]' not in body + + +def _salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: + """The yearly range from `compensation.summaryComponents` (needs + `?includeCompensation=true` on the feed URL, see `feed_url`): the first + `Salary` component, yearly or monthly (a monthly one multiplied by 12); + an hourly one is left out rather than annualised.""" + for component in components: + if component.get("compensationType") != "Salary": + continue + return yearly_amount(component.get("minValue"), component.get("maxValue"), + component.get("currencyCode"), component.get("interval"), + {"1 YEAR": 1, "1 MONTH": 12}) + return None, None, "" + + +def _salary_text(compensation: dict[str, Any]) -> str: + """One line per `summaryComponents` entry that states a bound, labelled + by the board's own `compensationType` when more than one does (a salary + range beside a bonus range, say); with just the one, the label is left + off, the same as Greenhouse's. `interval` is " " ("1 YEAR", + "1 HOUR"); the last word, lowered, is the period. With nothing to build + a line from, Ashby's own prose summary (`compensationTierSummary`, else + `scrapeableCompensationSalarySummary`) is what is left to show.""" + components = compensation.get("summaryComponents") or [] + stated = [c for c in components if num(c.get("minValue")) is not None + or num(c.get("maxValue")) is not None] + lines = [] + for component in stated: + period = str(component.get("interval") or "").split(" ")[-1].lower() + line = pay_line(num(component.get("minValue")), num(component.get("maxValue")), + component.get("currencyCode"), period) + if not line: + continue + if len(stated) > 1: + label = str(component.get("compensationType") or "").strip() + line = f"{label}: {line}" if label else line + lines.append(line) + if lines: + return "; ".join(lines) + return str(compensation.get("compensationTierSummary") + or compensation.get("scrapeableCompensationSalarySummary") or "") + + +def parse(body: bytes, slug: str) -> list[Row]: + data = json.loads(body) + rows = [] + for j in data.get("jobs") or []: + title = (j.get("title") or "").strip() + url = (j.get("jobUrl") or j.get("applyUrl") or "").strip() + if not title or not is_web_url(url): + continue + description = j.get("descriptionHtml") or j.get("descriptionPlain") or "" + compensation = j.get("compensation") or {} + salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or []) + rows.append({ + "source_id": str(j.get("id") or ""), + "title": title, + "location": (j.get("location") or "").strip(), + "url": url, + "posted": parse_dt(j.get("publishedAt")), + "department": (j.get("department") or j.get("team") or "").strip(), + "description": strip_html(description), + "remote": j.get("isRemote"), + "salary_text": _salary_text(compensation), + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/greenhouse.py b/plugins/atshosts/greenhouse.py new file mode 100644 index 0000000..de40e05 --- /dev/null +++ b/plugins/atshosts/greenhouse.py @@ -0,0 +1,138 @@ +"""Greenhouse: `greenhouse:`, the whole board in one JSON answer. + +The field map is ported from `openings.go` in +[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT). +""" + +from __future__ import annotations + +import html +import json +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line + +NAME = "greenhouse" +LABEL = "Greenhouse" +PATTERNS = ( + r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)", + r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)", +) + +_API = "boards-api.greenhouse.io" + + +def slug_ok(slug: str) -> bool: + return path_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_API) + + +def careers_url(slug: str) -> str: + return f"https://job-boards.greenhouse.io/{slug}" + + +def feed_url(slug: str) -> str: + return f"https://{_API}/v1/boards/{slug}/jobs?content=true&pay_transparency=true" + + +def fallback_url(slug: str) -> str: + """The feed without `content=true`, read once when the full one exceeds + the byte cap: the descriptions are lost, but the titles land rather than + the whole board being skipped.""" + return f"https://{_API}/v1/boards/{slug}/jobs" + + +def probe_url(slug: str) -> str: + return f"https://{_API}/v1/boards/{slug}" + + +def _salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: + """The yearly range from `pay_input_ranges` (needs `pay_transparency=true` + on the feed URL, see `feed_url`), which the list endpoint already carries + in the same request — no per-role fetch. Only when there is exactly one + range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations", + one per office) gives no way to tell which office the role in hand is + at, and picking one would misrepresent the columns. Greenhouse's own + ranges are cents, and yearly by convention except an hourly or a daily + one, named as such in `title` with no separate field to check — those + are skipped rather than annualised into a number nobody stated.""" + if len(ranges) != 1: + return None, None, "" + entry = ranges[0] + title = str(entry.get("title") or "").lower() + if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")): + return None, None, "" + code = str(entry.get("currency_type") or "").strip().upper() + if not code: + return None, None, "" + lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents")) + if lo is None and hi is None: + return None, None, "" + return (int(lo / 100) if lo is not None else None, + int(hi / 100) if hi is not None else None, code) + + +def _salary_text(ranges: list[dict[str, Any]]) -> str: + """One line per `pay_input_ranges` entry, joined with "; " — every range + Greenhouse states, not only the single one `_salary` accepts for the + columns. With more than one, each is labelled with the board's own + `title` ("Zone 1 Pay Range: ..."), since that is the only way to tell the + ranges apart; with just the one, the label adds nothing and is left off. + The hour/day check is the same one `_salary` uses to keep those out of + the yearly columns; here it only picks the word that follows "per".""" + multi = len(ranges) > 1 + lines = [] + for entry in ranges: + title = str(entry.get("title") or "").strip() + low_title = title.lower() + period = ("hour" if any(w in low_title for w in ("hour", "/hr")) + else "day" if "day" in low_title else "year") + lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents")) + line = pay_line(lo / 100 if lo is not None else None, + hi / 100 if hi is not None else None, + entry.get("currency_type"), period) + if not line: + continue + if multi: + label = title.rstrip(":").strip() or "Range" + line = f"{label}: {line}" + lines.append(line) + return "; ".join(lines) + + +def parse(body: bytes, slug: str) -> list[Row]: + data = json.loads(body) + rows = [] + for j in data.get("jobs") or []: + title = (j.get("title") or "").strip() + url = (j.get("absolute_url") or "").strip() + if not title or not is_web_url(url): + continue + departments = j.get("departments") or [] + # Greenhouse ships the description HTML-entity-escaped inside the + # JSON string (`<p>`), so it needs unescaping before the tags + # themselves can be stripped. + content = html.unescape(j.get("content") or "") + ranges = j.get("pay_input_ranges") or [] + salary_min, salary_max, currency = _salary(ranges) + rows.append({ + "source_id": str(j.get("id") or ""), + "title": title, + "location": ((j.get("location") or {}).get("name") or "").strip(), + "url": url, + "posted": parse_dt(j.get("first_published") or j.get("updated_at")), + "department": (departments[0].get("name") if departments else "") or "", + "description": strip_html(content), + "remote": None, # Greenhouse's feed does not say + "salary_text": _salary_text(ranges), + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/lever.py b/plugins/atshosts/lever.py new file mode 100644 index 0000000..7c33843 --- /dev/null +++ b/plugins/atshosts/lever.py @@ -0,0 +1,120 @@ +"""Lever: `lever:`, or `lever:eu/` for a board on Lever's EU +instance, the whole board in one JSON answer. + +Lever runs two instances that share nothing but the API: `api.lever.co` and +`api.eu.lever.co`, with the pages on `jobs.lever.co` and `jobs.eu.lever.co`. +A site exists on one of them only (TomTom and Prima answer 404 on the other, +2026-09-29), and the same name on both would be two companies, so the region +is part of the slug. It is a prefix a plain site name cannot carry, since a +Lever site name has no slash, which leaves every `lever:` target as it +was. + +The field map is ported from `openings.go` in +[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT). +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount + +NAME = "lever" +LABEL = "Lever" +#: Two groups on the EU hosts, `eu` and the site, which discovery joins into +#: the slug `eu/`. +PATTERNS = ( + r"jobs\.lever\.co/([A-Za-z0-9_-]+)", + r"api\.lever\.co/v0/postings/([A-Za-z0-9_-]+)", + r"jobs\.(eu)\.lever\.co/([A-Za-z0-9_-]+)", + r"api\.(eu)\.lever\.co/v0/postings/([A-Za-z0-9_-]+)", +) + +#: The region prefixes a slug may carry, to the infix of both host names. +_REGIONS = {"eu": "eu."} + + +def _split(slug: str) -> tuple[str, str]: + """`(host infix, site)`: `("eu.", "tomtom")` for `eu/tomtom`, `("", "acme")` + for `acme`.""" + region, sep, site = slug.partition("/") + return (_REGIONS.get(region, "?"), site) if sep else ("", slug) + + +def slug_ok(slug: str) -> bool: + infix, site = _split(slug) + return infix != "?" and path_slug_ok(site) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(f"api.{_split(slug)[0]}lever.co") + + +def careers_url(slug: str) -> str: + infix, site = _split(slug) + return f"https://jobs.{infix}lever.co/{site}" + + +def feed_url(slug: str) -> str: + infix, site = _split(slug) + return f"https://api.{infix}lever.co/v0/postings/{site}?mode=json" + + +def probe_url(slug: str) -> str: + infix, site = _split(slug) + return f"https://api.{infix}lever.co/v0/postings/{site}?limit=1&mode=json" + + +def probe_slugs(guess: str) -> tuple[str, ...]: + """A name is tried on both instances, the US one first, since a site that + names neither leaves no other way to tell which one it is on.""" + return (guess, *(f"{region}/{guess}" for region in _REGIONS)) + + +def _salary_text(salary_range: dict[str, Any]) -> str: + """One line from `salaryRange`, whatever its `interval` — including an + hourly or daily one, which `yearly_amount` drops from the columns. + Lever's interval is "per--" ("per-year-salary", + "per-hour-wage"); the middle word is the period.""" + interval = str(salary_range.get("interval") or "") + parts = interval.split("-") + period = parts[1] if len(parts) >= 3 else "" + return pay_line(num(salary_range.get("min")), num(salary_range.get("max")), + salary_range.get("currency"), period) + + +def parse(body: bytes, slug: str) -> list[Row]: + data = json.loads(body) + if not isinstance(data, list): + return [] + rows = [] + for j in data: + title = (j.get("text") or "").strip() + url = (j.get("hostedUrl") or "").strip() + if not title or not is_web_url(url): + continue + categories = j.get("categories") or {} + description = j.get("description") or j.get("descriptionPlain") or "" + salary_range = j.get("salaryRange") or {} + salary_min, salary_max, currency = yearly_amount( + salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"), + salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12}) + rows.append({ + "source_id": str(j.get("id") or ""), + "title": title, + "location": (categories.get("location") or "").strip(), + "url": url, + "posted": parse_dt(j.get("createdAt")), + "department": (categories.get("department") or categories.get("team") or "").strip(), + "description": strip_html(description), + "remote": (j.get("workplaceType") or "").strip() or None, + "salary_text": _salary_text(salary_range), + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/personio.py b/plugins/atshosts/personio.py new file mode 100644 index 0000000..e4a4c20 --- /dev/null +++ b/plugins/atshosts/personio.py @@ -0,0 +1,100 @@ +"""Personio: `personio:`, the whole board in one XML answer from +the company's own subdomain of jobs.personio.de. + +Written from one real board on 2026-09-23; jobhunt-hq did not read +Personio. +""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Callable + +from desk.sources import strip_html + +from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount + +NAME = "personio" +LABEL = "Personio" +PATTERNS = (r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)",) + +#: Personio's `salaryInformation/type` spellings, to the plain period word +#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through. +_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour", + "daily": "day", "weekly": "week"} + + +def slug_ok(slug: str) -> bool: + return label_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + """The company's own subdomain, so one company's slug cannot pass for + another's.""" + return only(f"{slug}.jobs.personio.de") + + +def careers_url(slug: str) -> str: + return f"https://{slug}.jobs.personio.de" + + +def feed_url(slug: str) -> str: + return f"https://{slug}.jobs.personio.de/xml" + + +def probe_url(slug: str) -> str: + return feed_url(slug) + + +def probe_ok(body: bytes) -> bool: + return b"" in body + + +def _salary_text(salary_info: ET.Element) -> str: + """One line from a `` element, whatever its `type`, + including an hourly or daily one that `yearly_amount` drops from the + columns.""" + period_raw = (salary_info.findtext("type") or "").strip().lower() + period = _PERIODS.get(period_raw, period_raw) + return pay_line(num(salary_info.findtext("min")), num(salary_info.findtext("max")), + salary_info.findtext("currencyCode"), period) + + +def parse(body: bytes, slug: str) -> list[Row]: + """`{slug}.jobs.personio.de/xml` -> `...`. No job + URL in the feed itself; Personio's own page is `/job/{id}`, confirmed + live against a real posting (2026-09-23).""" + root = ET.fromstring(body) + rows = [] + for position in root.findall("position"): + job_id = (position.findtext("id") or "").strip() + title = (position.findtext("name") or "").strip() + if not job_id or not title: + continue + description = " ".join( + (value.text or "") for value in position.findall("jobDescriptions/jobDescription/value")) + salary_info = position.find("salaryInformation") + salary_min = salary_max = None + currency = "" + salary_text = "" + if salary_info is not None: + salary_min, salary_max, currency = yearly_amount( + salary_info.findtext("min"), salary_info.findtext("max"), + salary_info.findtext("currencyCode"), salary_info.findtext("type"), + {"yearly": 1, "monthly": 12}) + salary_text = _salary_text(salary_info) + rows.append({ + "source_id": job_id, + "title": title, + "location": (position.findtext("office") or "").strip(), + "url": f"https://{slug}.jobs.personio.de/job/{job_id}", + "posted": parse_dt(position.findtext("createdAt")), + "department": (position.findtext("department") or "").strip(), + "description": strip_html(description), + "remote": "remote" in (position.findtext("office") or "").lower() or None, + "salary_text": salary_text, + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/recruitee.py b/plugins/atshosts/recruitee.py new file mode 100644 index 0000000..611ff09 --- /dev/null +++ b/plugins/atshosts/recruitee.py @@ -0,0 +1,83 @@ +"""Recruitee: `recruitee:`, the whole board in one JSON answer +from the company's own subdomain of recruitee.com. + +Written from one real board on 2026-09-23; jobhunt-hq did not read +Recruitee. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount + +NAME = "recruitee" +LABEL = "Recruitee" +PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",) + + +def slug_ok(slug: str) -> bool: + return label_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + """The company's own subdomain, so one company's slug cannot pass for + another's.""" + return only(f"{slug}.recruitee.com") + + +def careers_url(slug: str) -> str: + return f"https://{slug}.recruitee.com" + + +def feed_url(slug: str) -> str: + return f"https://{slug}.recruitee.com/api/offers/" + + +def probe_url(slug: str) -> str: + return feed_url(slug) + + +def probe_ok(body: bytes) -> bool: + return b'"offers":[' in body and b'"offers":[]' not in body + + +def _salary_text(salary: dict[str, Any]) -> str: + """One line from Recruitee's `salary` object; `period` is already a + plain word ("year", "month").""" + period = str(salary.get("period") or "").strip().lower() + return pay_line(num(salary.get("min")), num(salary.get("max")), + salary.get("currency"), period) + + +def parse(body: bytes, slug: str) -> list[Row]: + data = json.loads(body) + rows = [] + for j in data.get("offers") or []: + title = (j.get("title") or "").strip() + url = (j.get("careers_url") or "").strip() + if not title or not is_web_url(url): + continue + salary = j.get("salary") if isinstance(j.get("salary"), dict) else {} + salary_min, salary_max, currency = yearly_amount( + salary.get("min"), salary.get("max"), salary.get("currency"), + salary.get("period"), {"year": 1, "month": 12}) + rows.append({ + "source_id": str(j.get("id") or ""), + "title": title, + "location": (j.get("location") or j.get("city") or "").strip(), + "url": url, + "posted": parse_dt(j.get("published_at") or j.get("created_at")), + "department": (j.get("department") or "").strip(), + "description": strip_html(j.get("description") or ""), + "remote": j.get("remote"), + "salary_text": _salary_text(salary), + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, + }) + return rows diff --git a/plugins/atshosts/smartrecruiters.py b/plugins/atshosts/smartrecruiters.py new file mode 100644 index 0000000..5bd93c9 --- /dev/null +++ b/plugins/atshosts/smartrecruiters.py @@ -0,0 +1,89 @@ +"""SmartRecruiters: `smartrecruiters:`, paged by offset +with a stated total. The identifier is case-sensitive (`BoschGroup`). + +Written from one real board on 2026-09-23; jobhunt-hq did not read +SmartRecruiters. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok + +NAME = "smartrecruiters" +LABEL = "SmartRecruiters" +PATTERNS = ( + r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)", +) + +_API = "api.smartrecruiters.com" + +#: The page size asked for, which is also the most the host serves per page. +PAGE_SIZE = 100 + + +def slug_ok(slug: str) -> bool: + return path_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_API) + + +def careers_url(slug: str) -> str: + return f"https://jobs.smartrecruiters.com/{slug}" + + +def probe_url(slug: str) -> str: + return f"https://{_API}/v1/companies/{slug}/postings?limit=1" + + +def probe_ok(body: bytes) -> bool: + """An unknown identifier answers an empty listing, so a listing without + postings counts as no board.""" + return b'"totalFound":' in body and b'"totalFound":0' not in body + + +def page(get: Get, slug: str, cursor: Any) -> Page: + offset = int(cursor or 0) + data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings" + f"?offset={offset}&limit={PAGE_SIZE}")) + content = data.get("content") or [] + return data, content, data.get("totalFound"), offset + len(content) if content else None + + +def parse(body: bytes, slug: str) -> list[Row]: + return rows(json.loads(body), slug) + + +def rows(data: dict[str, Any], slug: str) -> list[Row]: + """One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings + -> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a + page, whatever `limit` asks for; the next is `?offset=` further on. The + list omits the job description (only the per-posting detail endpoint has + it, and fetching one per row would be hundreds of extra requests per + board), so `description` stays empty here.""" + out = [] + for j in data.get("content") or []: + title = (j.get("name") or "").strip() + job_id = str(j.get("id") or "") + if not title or not job_id: + continue + location = j.get("location") or {} + loc_text = (location.get("fullLocation") + or ", ".join(p for p in (location.get("city"), location.get("country")) if p)) + out.append({ + "source_id": job_id, + "title": title, + "location": (loc_text or "").strip(), + "url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}", + "posted": parse_dt(j.get("releasedDate")), + "department": ((j.get("department") or {}).get("label") or "").strip(), + "description": "", + "remote": location.get("remote"), + "salary_text": "", # SmartRecruiters' list endpoint states no pay at all + }) + return out diff --git a/plugins/atshosts/teamtailor.py b/plugins/atshosts/teamtailor.py new file mode 100644 index 0000000..281bd62 --- /dev/null +++ b/plugins/atshosts/teamtailor.py @@ -0,0 +1,136 @@ +"""Teamtailor: `teamtailor:` for acme.teamtailor.com, or, for a +career site on the company's own domain, that whole host name: +`teamtailor:careers.acme.com`. An RSS feed paged by offset that states no +total. + +Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor. +""" + +from __future__ import annotations + +import html +import re +import xml.etree.ElementTree as ET +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Get, Page, Row, is_web_url, only, parse_dt + +NAME = "teamtailor" +LABEL = "Teamtailor" +PATTERNS = ( + r"https?://([a-z0-9-]+)\.teamtailor\.com", + # No slug: a career site on the company's own domain, which only loads + # Teamtailor's CDN. Discovery reads its board off the page's host. + r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets", +) + +#: The page size, asked for as `per_page`. A feed with neither parameter +#: answers with every role (73 of 73 at Shine, 2026-09-29), but says nowhere +#: that it is the whole list, so this reads by page to an empty one; +#: `per_page=200` was answered in full too, and the cursor advances by what a +#: page returned, so a smaller cap on the host's side would not skip a role. +PAGE_SIZE = 50 + +#: Only a DNS name with a letter in its last label is a site, so an address +#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could +#: be pointed at; `ats._get` still refuses every host but this one, redirects +#: included. +_HOST = re.compile(r"^[a-z0-9-]+(\.[a-z0-9-]+)*$") + +#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`. +_TT = "{https://teamtailor.com/locations}" + +#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing +#: about where the role is worked and `temporary` is remote for a spell only, +#: so both stay unknown rather than become "onsite" or "remote". +_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"} + + +def site(slug: str) -> str: + """The host a career site is served from: `acme` is acme.teamtailor.com, + and a slug with a dot is a site on the company's own domain, + `careers.acme.com`, taken whole (the feed path is the same on both).""" + slug = slug.strip().lower() + return slug if "." in slug else f"{slug}.teamtailor.com" + + +def slug_ok(slug: str) -> bool: + host = site(slug) + return bool(slug.strip()) and bool(_HOST.match(host)) \ + and not host.rsplit(".", 1)[-1].isdigit() + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(site(slug)) + + +def careers_url(slug: str) -> str: + return f"https://{site(slug)}/jobs" + + +def probe_url(slug: str) -> str: + return f"https://{site(slug)}/jobs.rss" + + +def probe_ok(body: bytes) -> bool: + return b"" in body + + +def page(get: Get, slug: str, cursor: Any) -> Page: + offset = int(cursor or 0) + items = _items(get(f"https://{site(slug)}/jobs.rss?per_page={PAGE_SIZE}&offset={offset}")) + results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items] + return items, results, None, offset + len(items) if items else None + + +def parse(body: bytes, slug: str) -> list[Row]: + return rows(_items(body), slug) + + +def _items(body: bytes) -> list[ET.Element]: + """`{site}/jobs.rss` -> `...`, the feed every Teamtailor + career site serves (checked live on a dozen sites, 2026-09-29). A missing + `channel` is a page that is not a feed, and raises.""" + channel = ET.fromstring(body).find("channel") + if channel is None: + raise ValueError("not a Teamtailor feed: no channel") + return channel.findall("item") + + +def rows(items: list[ET.Element], slug: str) -> list[Row]: + """One page of items. The item has the title, the job URL (on the site's + own domain), a `guid` uuid, `pubDate`, the HTML `description` in full, + `remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the + `tt:` namespace, `locations` (a list: `city` and `country` each, `name` free + text), `department` and `role`, either of which may be empty. `place` joins + the distinct locations with "; ", which `country_of` reads a location at a + time. No pay is stated anywhere in the feed.""" + out = [] + for item in items: + title = (item.findtext("title") or "").strip() + url = (item.findtext("link") or "").strip() + if not title or not is_web_url(url): + continue + places: list[str] = [] + for loc in item.findall(f"{_TT}locations/{_TT}location"): + city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip() + country = (loc.findtext(f"{_TT}country") or "").strip() + place = city if country == city else ", ".join(p for p in (city, country) if p) + if place and place not in places: + places.append(place) + out.append({ + "source_id": (item.findtext("guid") or url).strip(), + "title": title, + "location": "; ".join(places), + "url": url, + "posted": parse_dt(item.findtext("pubDate")), + "department": ((item.findtext(f"{_TT}department") or "").strip() + or (item.findtext(f"{_TT}role") or "").strip()), + "description": html.unescape(strip_html(item.findtext("description") or "")), + "remote": _REMOTE.get((item.findtext("remoteStatus") or "").strip()), + "salary_text": "", # Teamtailor's feed states no pay at all + }) + return out diff --git a/plugins/atshosts/workable.py b/plugins/atshosts/workable.py new file mode 100644 index 0000000..6b2f0a6 --- /dev/null +++ b/plugins/atshosts/workable.py @@ -0,0 +1,84 @@ +"""Workable: `workable:`, paged ten roles at a time with a stated +total. + +Written from one real board, Hugging Face's, on 2026-09-28, when the first +board with roles was seen; jobhunt-hq did not read Workable. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok + +NAME = "workable" +LABEL = "Workable" +PATTERNS = ( + r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)", + r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com", +) + +_API = "apply.workable.com" + +# No probe: Workable's API sits behind a rate limit that a run of guesses +# trips, and a Workable site links its board plainly anyway. + + +def slug_ok(slug: str) -> bool: + return path_slug_ok(slug) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_API) + + +def careers_url(slug: str) -> str: + return f"https://{_API}/{slug}" + + +def page(get: Get, slug: str, cursor: Any) -> Page: + """One POST of the list; the next page is the same POST with + `{"token": nextPage}`, and the last page has no `nextPage`.""" + url = f"https://{_API}/api/v3/accounts/{slug}/jobs" + data = json.loads(get(url, method="POST", payload={"token": cursor} if cursor else {})) + return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None + + +def parse(body: bytes, slug: str) -> list[Row]: + return rows(json.loads(body), slug) + + +def rows(data: dict[str, Any], slug: str) -> list[Row]: + """One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs + -> `{"total": int, "results": [...], "nextPage": token}`, as Hugging + Face's board answered on 2026-09-28. A result carries no URL and no text: + the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and + the description is only on the per-job endpoint, one request per role, + so `description` stays empty here as it does for SmartRecruiters. + `location` is an object and `department` a list; `remote` is a flag + beside `workplace` (remote, hybrid, on_site).""" + out = [] + for j in data.get("results") or []: + title = (j.get("title") or "").strip() + shortcode = (j.get("shortcode") or "").strip() + if not title or not shortcode: + continue + loc = j.get("location") or {} + location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p) + dept = j.get("department") or "" + if isinstance(dept, list): + dept = ", ".join(d for d in dept if d) + out.append({ + "source_id": str(j.get("id") or shortcode), + "title": title, + "location": location, + "url": f"https://{_API}/{slug}/j/{shortcode}/", + "posted": parse_dt(j.get("published")), + "department": dept.strip(), + "description": "", + "remote": bool(j.get("remote")) or j.get("workplace") == "remote", + "salary_text": "", # Workable's list endpoint states no pay at all + }) + return out diff --git a/tests/test_ats.py b/tests/test_ats.py index 84adb4b..1a9ae27 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -19,6 +19,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats # noqa: E402 +from plugins.atshosts import smartrecruiters # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -384,7 +385,7 @@ def test_is_known_target_accepts_the_eight_hosts_and_refuses_others() -> None: "recruitee", "personio", "teamtailor"): assert ats.is_known_target(f"{host}:acme") assert not ats.is_known_target("linkedin:acme") - assert not ats.is_known_target("workday:acme") + assert not ats.is_known_target("taleo:acme") assert not ats.is_known_target("greenhouse:") assert not ats.is_known_target("nohostnoseparator") # a Teamtailor slug names a host: its own subdomain, or a whole domain @@ -420,7 +421,7 @@ def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None: assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com" assert ats.careers_url("teamtailor:acme") == "https://acme.teamtailor.com/jobs" assert ats.careers_url("teamtailor:careers.acme.com") == "https://careers.acme.com/jobs" - assert ats.careers_url("workday:acme") == "" + assert ats.careers_url("taleo:acme") == "" # --------------------------------------------------------------- body cap @@ -570,7 +571,7 @@ def test_smartrecruiters_pages_by_offset_to_total_found( urls.append(url) return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json") - monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2) monkeypatch.setattr(ats, "_get", fake_get) rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] @@ -588,7 +589,7 @@ def test_a_board_that_ends_short_of_its_total_is_incomplete( def fake_get(client, url, host, slug, *, method="GET", payload=None): return empty if "offset=2" in url else _page("smartrecruiters.json") - monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2) monkeypatch.setattr(ats, "_get", fake_get) rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] assert (len(rows), complete) == (2, False) @@ -604,7 +605,7 @@ def test_a_repeated_row_does_not_count_twice_toward_the_total( page["content"] = page["content"][1:] return json.dumps(page).encode() - monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2) monkeypatch.setattr(ats, "_get", fake_get) rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] assert (len(rows), complete) == (2, False)