diff --git a/docs/decisions.md b/docs/decisions.md index e1aa757..2059013 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -38,6 +38,15 @@ What was decided for roledesk and the one reason for each. The longer reasoning **The 8 MB cap and `?content=true` fight each other on the largest boards, and the cap won.** Both are named in the plan — the cap from openings.go, `content=true` so Greenhouse ships the description inline — but jobhunt-hq's own feeds never asked for content, so it never hit the size this combination produces. The real run (2026-09-23) refused seven boards outright for it: Anduril at 41 MB, and Anthropic, OpenAI, Databricks, Carvana, Axon and Airwallex between 9 and 19 MB, each because the biggest employers publish the most roles with the longest descriptions. Left as is, because raising the cap only moves the line and a company that big is exactly the case `openings.go`'s cap was written against; a real fix — retry without `content=true` when the capped response is refused, trading the description for at least the listing — is a follow-up, not a guess made here. +**The cap is raised to 96 MB, with one retry for Greenhouse.** The follow-up +the entry above deferred. Every URL `ats` fetches is on a fixed allow-list of +seven ATS API hosts, not an arbitrary one, so the cap openings.go wrote +against arbitrary URLs was the wrong number to port unchanged. 96 MB fits +every board seen so far, including Anduril's 41 MB; a board that still +exceeds it retries once without `?content=true` (Greenhouse only, since that +is the one host whose feed URL carries the flag), trading the description for +at least the listing. + **Workable's shape is confirmed, not a populated example.** Every slug tried against `apply.workable.com/api/v3/accounts/{slug}/jobs` during this build returned the documented `{"total": 0, "results": []}` before Cloudflare rate-limited further requests from this session, so the parser is written from Workable's long-published job object rather than from a fetched row. Worth a fixture-level recheck against a live board with open roles the next time this plugin is touched. ## Ranking diff --git a/plugins/ats.py b/plugins/ats.py index 9b47ab5..7751c73 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -5,14 +5,16 @@ unauthenticated JSON (Personio: XML) — first-party, no scraping, and a role that drops off the board is *closed*, which is the cheapest staleness check there is. -The field map, the redirect allow-list, the 8 MB body cap and the 10 s -timeout are ported from `openings.go` and `cmd/seedgen/main.go` in +The field map, the redirect allow-list and the 10 s timeout are ported from +`openings.go` and `cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including `classify`'s nine functions and their keywords. Workable, SmartRecruiters, Recruitee and Personio are new: jobhunt-hq only read Greenhouse, Lever and Ashby, so those four are written from one real board of each host, fetched while this file was written (2026-09-23), rather than from the Go source. +The body cap is not a straight port — see `MAX_BYTES` below. + A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:acme`. Every board is fetched independently; one that fails or times out is logged and skipped, and the rest continue. @@ -38,10 +40,17 @@ from desk.sources import ( log = logging.getLogger(__name__) -#: Matches openings.go: a board answers in 10s or it is skipped, and its body -#: is capped at 8 MB so a misbehaving host cannot hang a pull or fill memory. +#: Matches openings.go: a board answers in 10s or it is skipped. openings.go's +#: 8 MB body cap does not: it was written against arbitrary URLs, and every +#: URL fetched here is `?content=true` against a fixed allow-list of seven ATS +#: API hosts. The real run of 2026-09-23 hit it on the biggest employers — +#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and +#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised +#: to 96 MB so those boards fit; a board that still exceeds it gets one retry +#: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least +#: the titles land. TIMEOUT = 10.0 -MAX_BYTES = 8 << 20 +MAX_BYTES = 96 << 20 #: host -> (feed URL template, careers URL template). `{slug}` is filled in; #: recruitee and personio put the slug in the hostname instead of the path. @@ -102,6 +111,12 @@ class _Refused(Exception): """A board answered somewhere its allow-list does not cover.""" +class _TooLarge(ValueError): + """A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only + knows the old shape still catches it; `_fetch_rows` catches it by type to + retry Greenhouse without `?content=true`.""" + + def _get(client: httpx.Client, url: str, host: str, slug: str, *, method: str = "GET") -> bytes: """Fetch one board feed: https only, the host's own allow-list, at most @@ -127,7 +142,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, response.raise_for_status() body = response.content if len(body) > MAX_BYTES: - raise ValueError(f"response too large ({len(body)} bytes)") + raise _TooLarge(f"response too large ({len(body)} bytes)") return body raise ValueError("too many redirects") @@ -455,12 +470,24 @@ def careers_url(target: str) -> str: def _fetch_rows(client: httpx.Client, target: str) -> list[Row]: """One board, fetched and parsed. Raises on any failure — refused host, - timeout, a feed shape none of the parsers understand.""" + timeout, a feed shape none of the parsers understand. + + A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES` + gets one retry without it: the description is lost, but the titles land + rather than the whole board being skipped.""" if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") host, _, slug = target.partition(":") url = _BOARDS[host][0].format(slug=slug) - body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET") + try: + body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET") + except _TooLarge: + if host != "greenhouse" or "?" not in url: + raise + log.warning("ats: %s: body exceeded %d bytes with content=true; " + "retrying without it", target, MAX_BYTES) + url = url.split("?", 1)[0] + body = _get(client, url, host, slug, method="GET") return _PARSERS[host](body, slug) diff --git a/tests/test_ats.py b/tests/test_ats.py index a56437c..4619ce0 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -7,6 +7,8 @@ from __future__ import annotations import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) @@ -132,3 +134,42 @@ def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None: assert ats.careers_url("ashby:acme") == "https://jobs.ashbyhq.com/acme" assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com" assert ats.careers_url("workday:acme") == "" + + +# --------------------------------------------------------------- body cap + +def test_max_bytes_is_96_mb() -> None: + """Raised from openings.go's 8 MB: every URL here is on a fixed allow-list + of ATS API hosts, not an arbitrary one, and the real run of 2026-09-23 + refused boards up to 41 MB under it.""" + assert ats.MAX_BYTES == 96 << 20 + + +def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body( + monkeypatch: pytest.MonkeyPatch) -> None: + calls = [] + + def fake_get(client, url, host, slug, *, method="GET"): + calls.append(url) + if "?content=true" in url: + raise ats._TooLarge("response too large (100000000 bytes)") + return (FIXTURES / "greenhouse.json").read_bytes() + + monkeypatch.setattr(ats, "_get", fake_get) + rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type] + + assert calls == [ + "https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true", + "https://boards-api.greenhouse.io/v1/boards/acme/jobs", + ] + assert [r["source_id"] for r in rows] == ["1001", "1002"] + + +def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts( + monkeypatch: pytest.MonkeyPatch) -> None: + def fake_get(client, url, host, slug, *, method="GET"): + raise ats._TooLarge("response too large (100000000 bytes)") + + monkeypatch.setattr(ats, "_get", fake_get) + with pytest.raises(ats._TooLarge): + ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]