Paged boards are read to their stated total, and a board failing for 21 days closes
Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:<target>` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
0558ef19ed
commit
c7da12b93d
18 files changed
+695
-91
No files matched your search
+148
-22
@@ -18,7 +18,12 @@ The body cap is not a straight port — see `MAX_BYTES` below.
|
||||
|
||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||
`lever:acme`. Every board is fetched independently; one that fails or times
|
||||
out is logged and skipped, and the rest continue.
|
||||
out is logged, recorded as failing under `ats:failing:<target>` (see
|
||||
`ATSSource.fetch`), and skipped, and the rest continue.
|
||||
|
||||
Workable and SmartRecruiters page their list and state a total; every other
|
||||
host answers with the whole board at once. A paged board counts as complete
|
||||
only when it read as many distinct roles as the total says (see `_paged`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -27,6 +32,7 @@ import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
@@ -119,10 +125,11 @@ class _TooLarge(ValueError):
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET") -> bytes:
|
||||
method: str = "GET", payload: dict[str, Any] | None = None) -> bytes:
|
||||
"""Fetch one board feed: https only, the host's own allow-list, at most
|
||||
three redirect hops each checked the same way, and the body cut off at
|
||||
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`."""
|
||||
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none."""
|
||||
allow = _allowed_host(host, slug)
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
@@ -132,7 +139,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
response = client.request(
|
||||
method, current, timeout=TIMEOUT, follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
|
||||
json={} if method == "POST" else None,
|
||||
json=(payload or {}) if method == "POST" else None,
|
||||
)
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
@@ -258,15 +265,20 @@ def _parse_ashby(body: bytes) -> list[Row]:
|
||||
|
||||
|
||||
def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
||||
"""POST https://apply.workable.com/api/v3/accounts/{slug}/jobs with `{}` ->
|
||||
`{"total": int, "results": [...]}`, as Hugging Face's board answered on
|
||||
2026-09-28. A result carries no URL and no text: the job page is
|
||||
return _workable_rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
|
||||
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
|
||||
Face's board answered on 2026-09-28. Ten results a page; the next page is
|
||||
the same POST with `{"token": nextPage}`, and the last page has no
|
||||
`nextPage` (see `_paged`). A result carries no URL and no text: the job page is
|
||||
`https://apply.workable.com/{slug}/j/{shortcode}/`, and the description
|
||||
is only on the per-job endpoint, one request per role, so `description`
|
||||
stays empty here as it does for SmartRecruiters. `location` is an object
|
||||
and `department` a list; `remote` is a flag beside `workplace`
|
||||
(remote, hybrid, on_site)."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("results") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
@@ -293,11 +305,16 @@ def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
||||
|
||||
|
||||
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
|
||||
"""https://api.smartrecruiters.com/v1/companies/{slug}/postings ->
|
||||
`{"content": [...]}`. The list omits the job description (only the
|
||||
return _smartrecruiters_rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
|
||||
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
|
||||
page, whatever `limit` asks for; the next is `?offset=` further on (see
|
||||
`_paged`). The list omits the job description (only the
|
||||
per-posting detail endpoint has it, and fetching one per row would be
|
||||
hundreds of extra requests per board), so `description` stays empty here."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("content") or []:
|
||||
title = (j.get("name") or "").strip()
|
||||
@@ -474,9 +491,91 @@ def careers_url(target: str) -> str:
|
||||
return _BOARDS[host][1].format(slug=slug)
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
"""One board, fetched and parsed. Raises on any failure — refused host,
|
||||
timeout, a feed shape none of the parsers understand.
|
||||
#: Pages read from one paged board at most, and the pause between two of
|
||||
#: them. Workable answers ten roles a page and SmartRecruiters a hundred, so
|
||||
#: the cap is 1,000 and 10,000 roles; the pause keeps Workable, which answers
|
||||
#: a burst with 429, from seeing one.
|
||||
MAX_PAGES = 100
|
||||
PAGE_DELAY = 0.5
|
||||
|
||||
#: SmartRecruiters' page size, which is also the most it serves per page.
|
||||
SMARTRECRUITERS_PAGE = 100
|
||||
|
||||
#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its
|
||||
#: raw results, the total it states, and the cursor of the next page (None
|
||||
#: after the last). The first call gets cursor None.
|
||||
Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any]
|
||||
|
||||
|
||||
def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
||||
url = _BOARDS["workable"][0].format(slug=slug)
|
||||
data = json.loads(_get(client, url, "workable", slug, method="POST",
|
||||
payload={"token": cursor} if cursor else {}))
|
||||
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
|
||||
|
||||
|
||||
def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
url = (_BOARDS["smartrecruiters"][0].format(slug=slug)
|
||||
+ f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}")
|
||||
data = json.loads(_get(client, url, "smartrecruiters", slug))
|
||||
content = data.get("content") or []
|
||||
return data, content, data.get("totalFound"), offset + len(content) if content else None
|
||||
|
||||
|
||||
#: host -> (one page, that page's rows) for the hosts that page.
|
||||
_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page],
|
||||
Callable[[dict[str, Any], str], list[Row]]]] = {
|
||||
"workable": (_workable_page, _workable_rows),
|
||||
"smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows),
|
||||
}
|
||||
|
||||
|
||||
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""Every page of a Workable or SmartRecruiters board, `PAGE_DELAY` apart,
|
||||
until the host has no next page, the stated total is reached, or
|
||||
`MAX_PAGES`. Returns the rows and whether the board is complete: as many
|
||||
distinct results read as the host's stated total. Counted on the raw
|
||||
results, so a row the parser drops does not make a board incomplete, and
|
||||
on distinct ids, so a page that shifted under an offset and repeated a
|
||||
role does not make it complete.
|
||||
|
||||
A 429 after the first page stops the read with what it has, incomplete;
|
||||
one on the first page raises like any other failure."""
|
||||
host, _, slug = target.partition(":")
|
||||
page_of, rows_of = _PAGED[host]
|
||||
rows: dict[str, Row] = {}
|
||||
seen: set[str] = set()
|
||||
total: Any = None
|
||||
cursor: Any = None
|
||||
for page in range(MAX_PAGES):
|
||||
if page:
|
||||
time.sleep(PAGE_DELAY)
|
||||
try:
|
||||
data, results, total, cursor = page_of(client, slug, cursor)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if not page or exc.response.status_code != 429:
|
||||
raise
|
||||
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
|
||||
target, page + 1, len(seen), total)
|
||||
return list(rows.values()), False
|
||||
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||
for row in rows_of(data, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||
break
|
||||
complete = isinstance(total, int) and len(seen) == total
|
||||
if not complete:
|
||||
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), total)
|
||||
return list(rows.values()), complete
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""One board, fetched and parsed: its rows and whether they are the whole
|
||||
board. Raises on any failure — refused host, timeout, a feed shape none
|
||||
of the parsers understand. A host that does not page answers with the
|
||||
whole board, so it is complete whenever it parses; a paged one is
|
||||
complete when `_paged` says so.
|
||||
|
||||
A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES`
|
||||
gets one retry without it: the description is lost, but the titles land
|
||||
@@ -484,9 +583,11 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
host, _, slug = target.partition(":")
|
||||
if host in _PAGED:
|
||||
return _paged(client, target)
|
||||
url = _BOARDS[host][0].format(slug=slug)
|
||||
try:
|
||||
body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET")
|
||||
body = _get(client, url, host, slug, method="GET")
|
||||
except _TooLarge:
|
||||
if host != "greenhouse" or "?" not in url:
|
||||
raise
|
||||
@@ -494,18 +595,28 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
"retrying without it", target, MAX_BYTES)
|
||||
url = url.split("?", 1)[0]
|
||||
body = _get(client, url, host, slug, method="GET")
|
||||
return _PARSERS[host](body, slug)
|
||||
return _PARSERS[host](body, slug), True
|
||||
|
||||
|
||||
def fetch_board(target: str) -> list[Row]:
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||
"does this board answer, and what does it list". For `app companies
|
||||
add|check`, which ask that question about one board at a time outside any
|
||||
pull. Raises the same way `_fetch_rows` does."""
|
||||
"does this board answer, what does it list, and is that all of it". For
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. Raises the same way `_fetch_rows` does."""
|
||||
with httpx.Client() as client:
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
def failure_reason(exc: BaseException) -> str:
|
||||
"""One short line for why a board did not answer: the status of an HTTP
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
||||
return text[:200]
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
|
||||
class ATSSource(Source):
|
||||
@@ -541,11 +652,23 @@ class ATSSource(Source):
|
||||
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
rows = _fetch_rows(client, target)
|
||||
rows, complete = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
# The first failure in a row sets `since`, later ones only
|
||||
# the reason; `app.jobs.pull` closes the board's roles
|
||||
# once `since` is `STALE_DAYS` old.
|
||||
try:
|
||||
since = json.loads(self.state.get(failing, "") or "{}").get("since")
|
||||
except ValueError:
|
||||
since = None
|
||||
self.state[failing] = json.dumps({
|
||||
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"reason": failure_reason(exc)})
|
||||
continue
|
||||
self.state.pop(failing, None)
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
@@ -576,6 +699,9 @@ class ATSSource(Source):
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it.
|
||||
# has a partial id list treated as the whole of it. `complete`
|
||||
# says whether the ids are the whole board, which the pull
|
||||
# needs before it closes what they leave out.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
+48
-17
@@ -24,9 +24,12 @@ browser session of its own site) and Jooble (a key needs a manual
|
||||
application, and its API host answered a Cloudflare challenge).
|
||||
|
||||
A search result list is never a board's complete list, so a role missing from
|
||||
one run is not closed here. A search that ran out of results before its page
|
||||
limit was complete, and the pull closes what it stopped listing; the rest
|
||||
close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
||||
one run is not closed here. A search that read everything that matches was
|
||||
complete, and the pull closes what it stopped listing; the rest close once no
|
||||
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states
|
||||
how many results match, so an Adzuna search is complete when it read that
|
||||
many; Jobindex states no count, so a Jobindex search is complete when a page
|
||||
within the page limit came back short.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -124,20 +127,28 @@ def _text(fragment: str) -> str:
|
||||
|
||||
|
||||
def parse_jobindex(body: bytes) -> list[Row]:
|
||||
"""One page of a Jobindex RSS search. An item is `<title>` "Role, Company",
|
||||
return _jobindex_page(body)[0]
|
||||
|
||||
|
||||
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
|
||||
"""One page of a Jobindex RSS search, and how many items it had before
|
||||
any was dropped. An item is `<title>` "Role, Company",
|
||||
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
|
||||
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
|
||||
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
|
||||
("robot job") has no logo, so its company is what follows the title's
|
||||
last comma, or '' when there is none."""
|
||||
last comma, or '' when there is none. A body with no `<channel>` is not
|
||||
a search result and raises, since an empty list would read as a search
|
||||
that ran out and close everything it listed before."""
|
||||
root = ET.fromstring(body)
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
return []
|
||||
raise ValueError("jobindex answered without an RSS channel")
|
||||
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
||||
search_area = area_match.group(1).strip() if area_match else ""
|
||||
rows = []
|
||||
for item in channel.findall("item"):
|
||||
items = channel.findall("item")
|
||||
for item in items:
|
||||
link = (item.findtext("link") or "").strip()
|
||||
full_title = (item.findtext("title") or "").strip()
|
||||
if not full_title or not _web_url(link):
|
||||
@@ -169,21 +180,24 @@ def parse_jobindex(body: bytes) -> list[Row]:
|
||||
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
||||
"salary_min": None, "salary_max": None, "currency": "",
|
||||
})
|
||||
return rows
|
||||
return rows, len(items)
|
||||
|
||||
|
||||
def search_jobindex(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, newest first, until a short page or the page
|
||||
limit. Returns the rows and whether the results ran out before the limit."""
|
||||
limit. Returns the rows and whether the results ran out: a page with
|
||||
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
|
||||
a dropped item does not pass for the end. Jobindex states no total, so a
|
||||
search that read the page limit full is never complete."""
|
||||
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
|
||||
if params.get("area"):
|
||||
query["geoareaid"] = params["area"]
|
||||
rows: list[Row] = []
|
||||
for page in range(1, _pages(params) + 1):
|
||||
got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
||||
got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
||||
rows += got
|
||||
if len(got) < JOBINDEX_PAGE:
|
||||
if items < JOBINDEX_PAGE:
|
||||
return rows, True
|
||||
return rows, False
|
||||
|
||||
@@ -202,7 +216,10 @@ def parse_adzuna(body: bytes, country: str) -> list[Row]:
|
||||
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
|
||||
and snippets carry `<strong>` highlights. A salary Adzuna estimated
|
||||
(`salary_is_predicted`) is not the posting's, so it is dropped."""
|
||||
data = json.loads(body)
|
||||
return _adzuna_rows(json.loads(body), country)
|
||||
|
||||
|
||||
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
|
||||
rows = []
|
||||
for job in data.get("results") or []:
|
||||
title = _text(job.get("title") or "")
|
||||
@@ -237,6 +254,9 @@ def _parse_iso(value: Any) -> datetime | None:
|
||||
|
||||
def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, until a short page, the stated `count` or
|
||||
the page limit. Returns the rows and whether they are all `count` says
|
||||
match, or, on an answer with no `count`, whether a page came back short."""
|
||||
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
|
||||
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
|
||||
if not (app_id and app_key):
|
||||
@@ -252,13 +272,24 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
||||
if params.get("where"):
|
||||
query["where"] = params["where"]
|
||||
rows: list[Row] = []
|
||||
seen: set[str] = set()
|
||||
count: Any = None
|
||||
ran_out = False
|
||||
for page in range(1, _pages(params) + 1):
|
||||
url = ADZUNA_SEARCH.format(country=country, page=page)
|
||||
got = parse_adzuna(_get(client, url, query), country)
|
||||
rows += got
|
||||
if len(got) < ADZUNA_PAGE:
|
||||
return rows, True
|
||||
return rows, False
|
||||
data = json.loads(_get(client, url, query))
|
||||
results = data.get("results") or []
|
||||
count = data.get("count", count)
|
||||
seen |= {str(job.get("id") or "") for job in results}
|
||||
rows += _adzuna_rows(data, country)
|
||||
if len(results) < ADZUNA_PAGE:
|
||||
ran_out = True
|
||||
break
|
||||
if isinstance(count, int) and len(seen) >= count:
|
||||
break
|
||||
# Adzuna states how many results match; the search is complete when it
|
||||
# read that many distinct ones. Without a count, a short page is the end.
|
||||
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
|
||||
|
||||
|
||||
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
|
||||
|
||||
Reference in new issue
Block a user