Paged boards are read to their stated total, and a board failing for 21 days closes

Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:02:49 +02:00
1 parent 0558ef19ed
commit c7da12b93d
18 files changed
+695 -91

No files matched your search

+148 -22
View File
@@ -18,7 +18,12 @@ The body cap is not a straight port — see `MAX_BYTES` below.
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:acme`. Every board is fetched independently; one that fails or times
out is logged and skipped, and the rest continue.
out is logged, recorded as failing under `ats:failing:<target>` (see
`ATSSource.fetch`), and skipped, and the rest continue.
Workable and SmartRecruiters page their list and state a total; every other
host answers with the whole board at once. A paged board counts as complete
only when it read as many distinct roles as the total says (see `_paged`).
"""
from __future__ import annotations
@@ -27,6 +32,7 @@ import html
import json
import logging
import re
import time
import xml.etree.ElementTree as ET
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
@@ -119,10 +125,11 @@ class _TooLarge(ValueError):
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET") -> bytes:
method: str = "GET", payload: dict[str, Any] | None = None) -> bytes:
"""Fetch one board feed: https only, the host's own allow-list, at most
three redirect hops each checked the same way, and the body cut off at
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`."""
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none."""
allow = _allowed_host(host, slug)
current = url
for _hop in range(4):
@@ -132,7 +139,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
response = client.request(
method, current, timeout=TIMEOUT, follow_redirects=False,
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
json={} if method == "POST" else None,
json=(payload or {}) if method == "POST" else None,
)
if response.is_redirect:
location = response.headers.get("location") or ""
@@ -258,15 +265,20 @@ def _parse_ashby(body: bytes) -> list[Row]:
def _parse_workable(body: bytes, slug: str) -> list[Row]:
"""POST https://apply.workable.com/api/v3/accounts/{slug}/jobs with `{}` ->
`{"total": int, "results": [...]}`, as Hugging Face's board answered on
2026-09-28. A result carries no URL and no text: the job page is
return _workable_rows(json.loads(body), slug)
def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
Face's board answered on 2026-09-28. Ten results a page; the next page is
the same POST with `{"token": nextPage}`, and the last page has no
`nextPage` (see `_paged`). A result carries no URL and no text: the job page is
`https://apply.workable.com/{slug}/j/{shortcode}/`, and the description
is only on the per-job endpoint, one request per role, so `description`
stays empty here as it does for SmartRecruiters. `location` is an object
and `department` a list; `remote` is a flag beside `workplace`
(remote, hybrid, on_site)."""
data = json.loads(body)
rows = []
for j in data.get("results") or []:
title = (j.get("title") or "").strip()
@@ -293,11 +305,16 @@ def _parse_workable(body: bytes, slug: str) -> list[Row]:
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
"""https://api.smartrecruiters.com/v1/companies/{slug}/postings ->
`{"content": [...]}`. The list omits the job description (only the
return _smartrecruiters_rows(json.loads(body), slug)
def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
page, whatever `limit` asks for; the next is `?offset=` further on (see
`_paged`). The list omits the job description (only the
per-posting detail endpoint has it, and fetching one per row would be
hundreds of extra requests per board), so `description` stays empty here."""
data = json.loads(body)
rows = []
for j in data.get("content") or []:
title = (j.get("name") or "").strip()
@@ -474,9 +491,91 @@ def careers_url(target: str) -> str:
return _BOARDS[host][1].format(slug=slug)
def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
"""One board, fetched and parsed. Raises on any failure — refused host,
timeout, a feed shape none of the parsers understand.
#: Pages read from one paged board at most, and the pause between two of
#: them. Workable answers ten roles a page and SmartRecruiters a hundred, so
#: the cap is 1,000 and 10,000 roles; the pause keeps Workable, which answers
#: a burst with 429, from seeing one.
MAX_PAGES = 100
PAGE_DELAY = 0.5
#: SmartRecruiters' page size, which is also the most it serves per page.
SMARTRECRUITERS_PAGE = 100
#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its
#: raw results, the total it states, and the cursor of the next page (None
#: after the last). The first call gets cursor None.
Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any]
def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
url = _BOARDS["workable"][0].format(slug=slug)
data = json.loads(_get(client, url, "workable", slug, method="POST",
payload={"token": cursor} if cursor else {}))
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
offset = int(cursor or 0)
url = (_BOARDS["smartrecruiters"][0].format(slug=slug)
+ f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}")
data = json.loads(_get(client, url, "smartrecruiters", slug))
content = data.get("content") or []
return data, content, data.get("totalFound"), offset + len(content) if content else None
#: host -> (one page, that page's rows) for the hosts that page.
_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page],
Callable[[dict[str, Any], str], list[Row]]]] = {
"workable": (_workable_page, _workable_rows),
"smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows),
}
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""Every page of a Workable or SmartRecruiters board, `PAGE_DELAY` apart,
until the host has no next page, the stated total is reached, or
`MAX_PAGES`. Returns the rows and whether the board is complete: as many
distinct results read as the host's stated total. Counted on the raw
results, so a row the parser drops does not make a board incomplete, and
on distinct ids, so a page that shifted under an offset and repeated a
role does not make it complete.
A 429 after the first page stops the read with what it has, incomplete;
one on the first page raises like any other failure."""
host, _, slug = target.partition(":")
page_of, rows_of = _PAGED[host]
rows: dict[str, Row] = {}
seen: set[str] = set()
total: Any = None
cursor: Any = None
for page in range(MAX_PAGES):
if page:
time.sleep(PAGE_DELAY)
try:
data, results, total, cursor = page_of(client, slug, cursor)
except httpx.HTTPStatusError as exc:
if not page or exc.response.status_code != 429:
raise
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
target, page + 1, len(seen), total)
return list(rows.values()), False
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
for row in rows_of(data, slug):
rows.setdefault(row["source_id"], row)
if cursor is None or (isinstance(total, int) and len(seen) >= total):
break
complete = isinstance(total, int) and len(seen) == total
if not complete:
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), total)
return list(rows.values()), complete
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""One board, fetched and parsed: its rows and whether they are the whole
board. Raises on any failure — refused host, timeout, a feed shape none
of the parsers understand. A host that does not page answers with the
whole board, so it is complete whenever it parses; a paged one is
complete when `_paged` says so.
A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES`
gets one retry without it: the description is lost, but the titles land
@@ -484,9 +583,11 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
host, _, slug = target.partition(":")
if host in _PAGED:
return _paged(client, target)
url = _BOARDS[host][0].format(slug=slug)
try:
body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET")
body = _get(client, url, host, slug, method="GET")
except _TooLarge:
if host != "greenhouse" or "?" not in url:
raise
@@ -494,18 +595,28 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
"retrying without it", target, MAX_BYTES)
url = url.split("?", 1)[0]
body = _get(client, url, host, slug, method="GET")
return _PARSERS[host](body, slug)
return _PARSERS[host](body, slug), True
def fetch_board(target: str) -> list[Row]:
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
"does this board answer, and what does it list". For `app companies
add|check`, which ask that question about one board at a time outside any
pull. Raises the same way `_fetch_rows` does."""
"does this board answer, what does it list, and is that all of it". For
`app companies add|check`, which ask that question about one board at a
time outside any pull. Raises the same way `_fetch_rows` does."""
with httpx.Client() as client:
return _fetch_rows(client, target)
def failure_reason(exc: BaseException) -> str:
"""One short line for why a board did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
return text[:200]
# -------------------------------------------------------------- the plugin
class ATSSource(Source):
@@ -541,11 +652,23 @@ class ATSSource(Source):
for target, label in self.targets():
host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try:
rows = _fetch_rows(client, target)
rows, complete = _fetch_rows(client, target)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
# The first failure in a row sets `since`, later ones only
# the reason; `app.jobs.pull` closes the board's roles
# once `since` is `STALE_DAYS` old.
try:
since = json.loads(self.state.get(failing, "") or "{}").get("since")
except ValueError:
since = None
self.state[failing] = json.dumps({
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
"reason": failure_reason(exc)})
continue
self.state.pop(failing, None)
seen_ids = []
for row in rows:
@@ -576,6 +699,9 @@ class ATSSource(Source):
)
# Stamped only once the board has parsed without raising, so a
# board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it.
# has a partial id list treated as the whole of it. `complete`
# says whether the ids are the whole board, which the pull
# needs before it closes what they leave out.
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)