Stream a board's body against MAX_BYTES and a 60 s deadline; read timeout 30 s
_get streams the body with a running byte count, so an oversized board is refused at MAX_BYTES instead of after it is held whole, and bounds the whole request, redirects included, by DEADLINE, checked between chunks. The per-phase timeouts are httpx.Timeout(10, read=30), set from 84 live requests against mongodb, feedzai and the five Greenhouse boards with most open roles, with and without pay_transparency=true: the body arrives in under 1 s, the first byte waited up to 16.6 s, and the flag made no measurable difference. _paged works unchanged through it (tested over a mock transport, 429 included). Correct pull.py's BATCH_SIZE comment: a batch reads its boards in sequence, and the parallelism is the http pool's. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
2c4b992c50
commit
3abb44e10a
4 files changed
+126
-22
No files matched your search
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -279,6 +280,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
sent: list[int] = []
|
||||
|
||||
def chunks() -> Iterator[bytes]:
|
||||
for i in range(100):
|
||||
sent.append(i)
|
||||
yield b"x" * 10
|
||||
|
||||
monkeypatch.setattr(ats, "MAX_BYTES", 25)
|
||||
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
|
||||
pytest.raises(ats._TooLarge):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
assert len(sent) == 3 # refused on the third chunk, not the hundredth
|
||||
|
||||
|
||||
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
clock = iter(range(0, 1000, 10))
|
||||
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
|
||||
monkeypatch.setattr(ats, "DEADLINE", 25.0)
|
||||
body = (b"x" for _ in range(100))
|
||||
with _client(lambda request: httpx.Response(200, content=body)) as client, \
|
||||
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
|
||||
|
||||
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/v0/postings/old":
|
||||
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
|
||||
if request.url.path == "/v0/postings/away":
|
||||
return httpx.Response(302, headers={"location": "https://evil.example/"})
|
||||
return httpx.Response(200, content=b"[]")
|
||||
|
||||
with _client(handler) as client:
|
||||
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
|
||||
with pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
|
||||
|
||||
|
||||
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
|
||||
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
|
||||
429 on a later page still reaches it as an `HTTPStatusError`."""
|
||||
bodies: list[Any] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
payload = json.loads(request.content or b"{}")
|
||||
bodies.append(payload)
|
||||
if payload.get("token") == "stop":
|
||||
return httpx.Response(429)
|
||||
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
|
||||
if payload:
|
||||
page["total"], page["nextPage"] = 4, "stop"
|
||||
return httpx.Response(200, json=page)
|
||||
|
||||
with _client(handler) as client:
|
||||
rows, complete = ats._fetch_rows(client, "workable:exotec")
|
||||
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
|
||||
assert (len(rows), complete) == (3, False)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- paging
|
||||
|
||||
def _page(name: str) -> bytes:
|
||||
|
||||
Reference in new issue
Block a user