From 3abb44e10a9524a838e2900fa0de148c449961fc Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 15:12:14 +0200 Subject: [PATCH] Stream a board's body against MAX_BYTES and a 60 s deadline; read timeout 30 s _get streams the body with a running byte count, so an oversized board is refused at MAX_BYTES instead of after it is held whole, and bounds the whole request, redirects included, by DEADLINE, checked between chunks. The per-phase timeouts are httpx.Timeout(10, read=30), set from 84 live requests against mongodb, feedzai and the five Greenhouse boards with most open roles, with and without pay_transparency=true: the body arrives in under 1 s, the first byte waited up to 16.6 s, and the flag made no measurable difference. _paged works unchanged through it (tested over a mock transport, 429 included). Correct pull.py's BATCH_SIZE comment: a batch reads its boards in sequence, and the parallelism is the http pool's. Co-Authored-By: Claude Opus --- app/jobs/pull.py | 10 ++++--- docs/decisions.md | 2 ++ plugins/ats.py | 69 ++++++++++++++++++++++++++++++++++------------- tests/test_ats.py | 67 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 22 deletions(-) diff --git a/app/jobs/pull.py b/app/jobs/pull.py index d0038b7..45aa04d 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -47,9 +47,13 @@ from app.storage import Store log = logging.getLogger(__name__) -#: As many boards in the air per job as there are `http` workers by default -#: (see `config.yaml`), so one pull keeps the pool busy without one job -#: holding all 190-odd boards' worth of retries against a single attempt. +#: Targets per `pull_batch` job. A batch reads its boards one after another, +#: so the parallelism is the number of batches the `http` pool runs at once +#: (8 workers by default, see `config.yaml`), not this number. On the live +#: database of 2026-09-29, 175 boards and 4 searches make 19 batches, enough +#: to keep eight workers busy, while a batch that fails or is killed costs at +#: most ten targets' progress. A board's request is bounded by +#: `plugins.ats.DEADLINE` plus one read timeout. BATCH_SIZE = 10 SETTLE = "app.jobs.pull:close_boards" diff --git a/docs/decisions.md b/docs/decisions.md index 41736ae..cb4878a 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -78,6 +78,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **The body cap is 96 MB, with one retry for Greenhouse.** openings.go's 8 MB cap was written against arbitrary URLs, while `ats` only fetches a fixed allow-list of seven ATS hosts, and the real run found `?content=true` boards between 9 and 41 MB. A board that still exceeds 96 MB retries once without `?content=true`, trading the description for the listing. +**A board request is streamed, with a 30 s read timeout and a 60 s deadline.** `_get` counts the body as it arrives and refuses it at `MAX_BYTES`, where reading `response.content` first held the whole body in memory before the check. httpx's timeouts are per phase, so a board that keeps sending a byte now and then never timed out; the deadline covers the whole request, redirects and body included, and is checked between chunks. openings.go's 10 s read timeout failed `greenhouse:mongodb` and `greenhouse:feedzai` in a pull of 2026-09-28. Measured on 2026-09-29, three runs each of those two and the five Greenhouse boards with the most open roles, with and without `pay_transparency=true` (84 requests, every one a CloudFront miss): a body of up to 10.4 MB arrives in under a second once it starts, and the first byte waited up to 16.6 s, with the three slowest at 16.6, 14.4 and 10.9 s. The flag made no measurable difference; the slow first byte is the origin's, with or without it. A read may wait 30 s, about twice the worst seen, and the deadline is 60 s, which leaves a large board room on a slower link. + **Workable's list endpoint has no URL and no text.** The parser was first written to Workable's published job object, since every slug tried in the build answered an empty list; the first populated board (Hugging Face, 2026-09-28) showed the v3 shape instead: `location` an object, `department` a list, a `shortcode` and no URL, and no description. The job page is `apply.workable.com/{slug}/j/{shortcode}/`, and `description` stays empty as for SmartRecruiters rather than costing one request per role. **Workable and SmartRecruiters are read to the end, and a board closes roles only when it read the total it states.** Workable answers 10 roles a page with a `nextPage` token and SmartRecruiters at most 100 a page by `offset`, and both parsers read page one only, so the settle took page one for the whole board and closed what fell off it: Delivery Hero stated 980 roles and 100 were open. `_paged` follows the pages 0.5 s apart, up to 100 pages, and the plugin stamps `ats:complete` only when the distinct ids read equal `total` or `totalFound`. The count is taken on the raw results, so a row the parser drops does not block closure, and on distinct ids, so a role repeated by a shifted offset does not fake completeness. A 429 after page one ends the read: the roles read are stored and nothing closes, because a partial list is no evidence that the rest closed. A 429 on page one is a failure like any other. The other five hosts answer with the whole board and are complete whenever they parse. diff --git a/plugins/ats.py b/plugins/ats.py index 334f386..842d86b 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -5,7 +5,7 @@ unauthenticated JSON (Personio: XML) — first-party, no scraping, and a role that drops off the board is *closed*, which is the cheapest staleness check there is. -The field map, the redirect allow-list and the 10 s timeout are ported from +The field map and the redirect allow-list are ported from `openings.go` and `cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including `classify`'s nine functions and their keywords. Workable, SmartRecruiters, @@ -47,8 +47,7 @@ from desk.sources import ( log = logging.getLogger(__name__) -#: Matches openings.go: a board answers in 10s or it is skipped. openings.go's -#: 8 MB body cap does not: it was written against arbitrary URLs, and every +#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every #: URL fetched here is `?content=true` against a fixed allow-list of seven ATS #: API hosts. The real run of 2026-09-23 hit it on the biggest employers — #: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and @@ -56,9 +55,26 @@ log = logging.getLogger(__name__) #: to 96 MB so those boards fit; a board that still exceeds it gets one retry #: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least #: the titles land. -TIMEOUT = 10.0 MAX_BYTES = 96 << 20 +#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai` +#: failed a pull of 2026-09-28 with a read timeout. Measured live on +#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse +#: boards with the most open roles (databricks, stripe, anthropic, feverup, +#: datadog), with and without `pay_transparency=true`, cached and with a +#: cache-busting parameter (84 requests; CloudFront answered every one +#: "Miss", so all reached the origin): the body arrives in under 1 s once it +#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the +#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic, +#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise. +#: The flag made no difference that three runs could tell apart from that +#: noise. So a read may wait 30 s, about twice the worst first byte seen, +#: and a whole request, redirects and body included, 60 s (see `_get`), +#: about four times the slowest seen, which still leaves a 96 MB body room +#: on a link far slower than the one measured. +TIMEOUTS = httpx.Timeout(10.0, read=30.0) +DEADLINE = 60.0 + #: host -> (feed URL template, careers URL template). `{slug}` is filled in; #: recruitee and personio put the slug in the hostname instead of the path. _BOARDS: dict[str, tuple[str, str]] = { @@ -141,29 +157,44 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, """Fetch one board feed: https only, the host's own allow-list, at most three redirect hops each checked the same way, and the body cut off at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST - sends `payload` as JSON, `{}` when there is none.""" + sends `payload` as JSON, `{}` when there is none. + + The body is streamed and counted as it arrives, so an oversized one is + refused at `MAX_BYTES` rather than after it has been read whole, and + the whole request, redirects included, must end within `DEADLINE`: the + per-read `TIMEOUTS` alone let a board that trickles bytes run on for + good. The deadline is checked between chunks, so it can be overrun by + at most one read timeout.""" allow = _allowed_host(host, slug) + deadline = time.monotonic() + DEADLINE current = url for _hop in range(4): parsed = httpx.URL(current) if parsed.scheme != "https" or not allow(parsed.host): raise _Refused(f"{current} is not an allowed host for {host}") - response = client.request( - method, current, timeout=TIMEOUT, follow_redirects=False, + with client.stream( + method, current, timeout=TIMEOUTS, follow_redirects=False, headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"}, json=(payload or {}) if method == "POST" else None, - ) - if response.is_redirect: - location = response.headers.get("location") or "" - if not location: - raise ValueError("redirect with no location") - current = str(httpx.URL(current).join(location)) - continue - response.raise_for_status() - body = response.content - if len(body) > MAX_BYTES: - raise _TooLarge(f"response too large ({len(body)} bytes)") - return body + ) as response: + if response.is_redirect: + location = response.headers.get("location") or "" + if not location: + raise ValueError("redirect with no location") + current = str(httpx.URL(current).join(location)) + continue + response.raise_for_status() + chunks: list[bytes] = [] + size = 0 + for chunk in response.iter_bytes(): + size += len(chunk) + if size > MAX_BYTES: + raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)") + if time.monotonic() > deadline: + raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s", + request=response.request) + chunks.append(chunk) + return b"".join(chunks) raise ValueError("too many redirects") diff --git a/tests/test_ats.py b/tests/test_ats.py index a10f426..0823ee2 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -7,6 +7,7 @@ from __future__ import annotations import json import sys from pathlib import Path +from collections.abc import Callable, Iterator from typing import Any import httpx @@ -279,6 +280,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts( ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type] +def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def test_get_stops_reading_at_max_bytes_rather_than_after_the_body( + monkeypatch: pytest.MonkeyPatch) -> None: + sent: list[int] = [] + + def chunks() -> Iterator[bytes]: + for i in range(100): + sent.append(i) + yield b"x" * 10 + + monkeypatch.setattr(ats, "MAX_BYTES", 25) + with _client(lambda request: httpx.Response(200, content=chunks())) as client, \ + pytest.raises(ats._TooLarge): + ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme") + assert len(sent) == 3 # refused on the third chunk, not the hundredth + + +def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive( + monkeypatch: pytest.MonkeyPatch) -> None: + clock = iter(range(0, 1000, 10)) + monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock))) + monkeypatch.setattr(ats, "DEADLINE", 25.0) + body = (b"x" for _ in range(100)) + with _client(lambda request: httpx.Response(200, content=body)) as client, \ + pytest.raises(httpx.ReadTimeout, match="within 25 s"): + ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme") + + +def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v0/postings/old": + return httpx.Response(301, headers={"location": "/v0/postings/acme"}) + if request.url.path == "/v0/postings/away": + return httpx.Response(302, headers={"location": "https://evil.example/"}) + return httpx.Response(200, content=b"[]") + + with _client(handler) as client: + assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]" + with pytest.raises(ats._Refused): + ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme") + + +def test_paging_reads_through_the_streamed_get(no_delay: None) -> None: + """`_paged` over the real `_get`: the POST body carries the cursor, and a + 429 on a later page still reaches it as an `HTTPStatusError`.""" + bodies: list[Any] = [] + + def handler(request: httpx.Request) -> httpx.Response: + payload = json.loads(request.content or b"{}") + bodies.append(payload) + if payload.get("token") == "stop": + return httpx.Response(429) + page = json.loads(_page("workable-2.json" if payload else "workable.json")) + if payload: + page["total"], page["nextPage"] = 4, "stop" + return httpx.Response(200, json=page) + + with _client(handler) as client: + rows, complete = ats._fetch_rows(client, "workable:exotec") + assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}] + assert (len(rows), complete) == (3, False) + + # ----------------------------------------------------------------- paging def _page(name: str) -> bytes: