Merge branch 'r2-backend'

# Conflicts:
#	app/storage/schema.py
This commit is contained in:
Wolf committed 2026-09-29 15:22:03 +02:00
commit 562fa3ab09
15 files changed
+924 -157

No files matched your search

+67
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import json
import sys
from pathlib import Path
from collections.abc import Callable, Iterator
from typing import Any
import httpx
@@ -333,6 +334,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
return httpx.Client(transport=httpx.MockTransport(handler))
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
monkeypatch: pytest.MonkeyPatch) -> None:
sent: list[int] = []
def chunks() -> Iterator[bytes]:
for i in range(100):
sent.append(i)
yield b"x" * 10
monkeypatch.setattr(ats, "MAX_BYTES", 25)
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
pytest.raises(ats._TooLarge):
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
assert len(sent) == 3 # refused on the third chunk, not the hundredth
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
monkeypatch: pytest.MonkeyPatch) -> None:
clock = iter(range(0, 1000, 10))
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
monkeypatch.setattr(ats, "DEADLINE", 25.0)
body = (b"x" for _ in range(100))
with _client(lambda request: httpx.Response(200, content=body)) as client, \
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
def handler(request: httpx.Request) -> httpx.Response:
if request.url.path == "/v0/postings/old":
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
if request.url.path == "/v0/postings/away":
return httpx.Response(302, headers={"location": "https://evil.example/"})
return httpx.Response(200, content=b"[]")
with _client(handler) as client:
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
with pytest.raises(ats._Refused):
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
429 on a later page still reaches it as an `HTTPStatusError`."""
bodies: list[Any] = []
def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content or b"{}")
bodies.append(payload)
if payload.get("token") == "stop":
return httpx.Response(429)
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
if payload:
page["total"], page["nextPage"] = 4, "stop"
return httpx.Response(200, json=page)
with _client(handler) as client:
rows, complete = ats._fetch_rows(client, "workable:exotec")
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
assert (len(rows), complete) == (3, False)
# ----------------------------------------------------------------- paging
def _page(name: str) -> bytes: