Merge branch 'worktree-agent-ac3c9345edd29a1c6'
# Conflicts: # app/cli.py # app/jobs/sources.py # app/static/app.css # app/static/companies.js # app/web/api.py # tests/test_web.py
This commit is contained in:
commit
c6246af994
21 files changed
+1017
-155
No files matched your search
+154
-26
@@ -18,7 +18,12 @@ The body cap is not a straight port — see `MAX_BYTES` below.
|
||||
|
||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||
`lever:acme`. Every board is fetched independently; one that fails or times
|
||||
out is logged and skipped, and the rest continue.
|
||||
out is logged, recorded as failing under `ats:failing:<target>` (see
|
||||
`ATSSource.fetch`), and skipped, and the rest continue.
|
||||
|
||||
Workable and SmartRecruiters page their list and state a total; every other
|
||||
host answers with the whole board at once. A paged board counts as complete
|
||||
only when it read as many distinct roles as the total says (see `_paged`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -27,6 +32,7 @@ import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
@@ -129,10 +135,11 @@ class _TooLarge(ValueError):
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET") -> bytes:
|
||||
method: str = "GET", payload: dict[str, Any] | None = None) -> bytes:
|
||||
"""Fetch one board feed: https only, the host's own allow-list, at most
|
||||
three redirect hops each checked the same way, and the body cut off at
|
||||
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`."""
|
||||
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none."""
|
||||
allow = _allowed_host(host, slug)
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
@@ -142,7 +149,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
response = client.request(
|
||||
method, current, timeout=TIMEOUT, follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
|
||||
json={} if method == "POST" else None,
|
||||
json=(payload or {}) if method == "POST" else None,
|
||||
)
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
@@ -268,15 +275,20 @@ def _parse_ashby(body: bytes) -> list[Row]:
|
||||
|
||||
|
||||
def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
||||
"""POST https://apply.workable.com/api/v3/accounts/{slug}/jobs with `{}` ->
|
||||
`{"total": int, "results": [...]}`, as Hugging Face's board answered on
|
||||
2026-09-28. A result carries no URL and no text: the job page is
|
||||
return _workable_rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
|
||||
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
|
||||
Face's board answered on 2026-09-28. Ten results a page; the next page is
|
||||
the same POST with `{"token": nextPage}`, and the last page has no
|
||||
`nextPage` (see `_paged`). A result carries no URL and no text: the job page is
|
||||
`https://apply.workable.com/{slug}/j/{shortcode}/`, and the description
|
||||
is only on the per-job endpoint, one request per role, so `description`
|
||||
stays empty here as it does for SmartRecruiters. `location` is an object
|
||||
and `department` a list; `remote` is a flag beside `workplace`
|
||||
(remote, hybrid, on_site)."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("results") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
@@ -303,11 +315,16 @@ def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
||||
|
||||
|
||||
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
|
||||
"""https://api.smartrecruiters.com/v1/companies/{slug}/postings ->
|
||||
`{"content": [...]}`. The list omits the job description (only the
|
||||
return _smartrecruiters_rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
|
||||
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
|
||||
page, whatever `limit` asks for; the next is `?offset=` further on (see
|
||||
`_paged`). The list omits the job description (only the
|
||||
per-posting detail endpoint has it, and fetching one per row would be
|
||||
hundreds of extra requests per board), so `description` stays empty here."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("content") or []:
|
||||
title = (j.get("name") or "").strip()
|
||||
@@ -484,9 +501,91 @@ def careers_url(target: str) -> str:
|
||||
return _BOARDS[host][1].format(slug=slug)
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
"""One board, fetched and parsed. Raises on any failure — refused host,
|
||||
timeout, a feed shape none of the parsers understand.
|
||||
#: Pages read from one paged board at most, and the pause between two of
|
||||
#: them. Workable answers ten roles a page and SmartRecruiters a hundred, so
|
||||
#: the cap is 1,000 and 10,000 roles; the pause keeps Workable, which answers
|
||||
#: a burst with 429, from seeing one.
|
||||
MAX_PAGES = 100
|
||||
PAGE_DELAY = 0.5
|
||||
|
||||
#: SmartRecruiters' page size, which is also the most it serves per page.
|
||||
SMARTRECRUITERS_PAGE = 100
|
||||
|
||||
#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its
|
||||
#: raw results, the total it states, and the cursor of the next page (None
|
||||
#: after the last). The first call gets cursor None.
|
||||
Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any]
|
||||
|
||||
|
||||
def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
||||
url = _BOARDS["workable"][0].format(slug=slug)
|
||||
data = json.loads(_get(client, url, "workable", slug, method="POST",
|
||||
payload={"token": cursor} if cursor else {}))
|
||||
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
|
||||
|
||||
|
||||
def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
url = (_BOARDS["smartrecruiters"][0].format(slug=slug)
|
||||
+ f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}")
|
||||
data = json.loads(_get(client, url, "smartrecruiters", slug))
|
||||
content = data.get("content") or []
|
||||
return data, content, data.get("totalFound"), offset + len(content) if content else None
|
||||
|
||||
|
||||
#: host -> (one page, that page's rows) for the hosts that page.
|
||||
_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page],
|
||||
Callable[[dict[str, Any], str], list[Row]]]] = {
|
||||
"workable": (_workable_page, _workable_rows),
|
||||
"smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows),
|
||||
}
|
||||
|
||||
|
||||
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""Every page of a Workable or SmartRecruiters board, `PAGE_DELAY` apart,
|
||||
until the host has no next page, the stated total is reached, or
|
||||
`MAX_PAGES`. Returns the rows and whether the board is complete: as many
|
||||
distinct results read as the host's stated total. Counted on the raw
|
||||
results, so a row the parser drops does not make a board incomplete, and
|
||||
on distinct ids, so a page that shifted under an offset and repeated a
|
||||
role does not make it complete.
|
||||
|
||||
A 429 after the first page stops the read with what it has, incomplete;
|
||||
one on the first page raises like any other failure."""
|
||||
host, _, slug = target.partition(":")
|
||||
page_of, rows_of = _PAGED[host]
|
||||
rows: dict[str, Row] = {}
|
||||
seen: set[str] = set()
|
||||
total: Any = None
|
||||
cursor: Any = None
|
||||
for page in range(MAX_PAGES):
|
||||
if page:
|
||||
time.sleep(PAGE_DELAY)
|
||||
try:
|
||||
data, results, total, cursor = page_of(client, slug, cursor)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if not page or exc.response.status_code != 429:
|
||||
raise
|
||||
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
|
||||
target, page + 1, len(seen), total)
|
||||
return list(rows.values()), False
|
||||
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||
for row in rows_of(data, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||
break
|
||||
complete = isinstance(total, int) and len(seen) == total
|
||||
if not complete:
|
||||
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), total)
|
||||
return list(rows.values()), complete
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""One board, fetched and parsed: its rows and whether they are the whole
|
||||
board. Raises on any failure — refused host, timeout, a feed shape none
|
||||
of the parsers understand. A host that does not page answers with the
|
||||
whole board, so it is complete whenever it parses; a paged one is
|
||||
complete when `_paged` says so.
|
||||
|
||||
A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES`
|
||||
gets one retry without it: the description is lost, but the titles land
|
||||
@@ -494,9 +593,11 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
host, _, slug = target.partition(":")
|
||||
if host in _PAGED:
|
||||
return _paged(client, target)
|
||||
url = _BOARDS[host][0].format(slug=slug)
|
||||
try:
|
||||
body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET")
|
||||
body = _get(client, url, host, slug, method="GET")
|
||||
except _TooLarge:
|
||||
if host != "greenhouse" or "?" not in url:
|
||||
raise
|
||||
@@ -504,18 +605,28 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
"retrying without it", target, MAX_BYTES)
|
||||
url = url.split("?", 1)[0]
|
||||
body = _get(client, url, host, slug, method="GET")
|
||||
return _PARSERS[host](body, slug)
|
||||
return _PARSERS[host](body, slug), True
|
||||
|
||||
|
||||
def fetch_board(target: str) -> list[Row]:
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||
"does this board answer, and what does it list". For `app companies
|
||||
add|check`, which ask that question about one board at a time outside any
|
||||
pull. Raises the same way `_fetch_rows` does."""
|
||||
"does this board answer, what does it list, and is that all of it". For
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. Raises the same way `_fetch_rows` does."""
|
||||
with httpx.Client() as client:
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
def failure_reason(exc: BaseException) -> str:
|
||||
"""One short line for why a board did not answer: the status of an HTTP
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
||||
return text[:200]
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
|
||||
class ATSSource(Source):
|
||||
@@ -543,19 +654,33 @@ class ATSSource(Source):
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
with httpx.Client() as client:
|
||||
# Written by the pull job before any board is fetched (see
|
||||
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry
|
||||
# run — has no run to stamp itself against, so it makes one up.
|
||||
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
|
||||
# Handed in by the batch that runs this plugin, as the marker of
|
||||
# the pull run that queued it (see `app/jobs/pull.py`), and never
|
||||
# read from state, where a later run may have replaced it. A
|
||||
# plugin run on its own — a test, a dry run — has no run to stamp
|
||||
# itself against, so it makes one up.
|
||||
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
rows = _fetch_rows(client, target)
|
||||
rows, complete = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
# The first failure in a row sets `since`, later ones only
|
||||
# the reason; `app.jobs.pull` closes the board's roles
|
||||
# once `since` is `STALE_DAYS` old.
|
||||
try:
|
||||
since = json.loads(self.state.get(failing, "") or "{}").get("since")
|
||||
except ValueError:
|
||||
since = None
|
||||
self.state[failing] = json.dumps({
|
||||
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"reason": failure_reason(exc)})
|
||||
continue
|
||||
self.state.pop(failing, None)
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
@@ -586,6 +711,9 @@ class ATSSource(Source):
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it.
|
||||
# has a partial id list treated as the whole of it. `complete`
|
||||
# says whether the ids are the whole board, which the pull
|
||||
# needs before it closes what they leave out.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
Reference in new issue
Block a user