Merge branch 'r2-backend'
# Conflicts: # app/storage/schema.py
This commit is contained in:
commit
562fa3ab09
15 files changed
+924
-157
No files matched your search
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -333,6 +334,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
sent: list[int] = []
|
||||
|
||||
def chunks() -> Iterator[bytes]:
|
||||
for i in range(100):
|
||||
sent.append(i)
|
||||
yield b"x" * 10
|
||||
|
||||
monkeypatch.setattr(ats, "MAX_BYTES", 25)
|
||||
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
|
||||
pytest.raises(ats._TooLarge):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
assert len(sent) == 3 # refused on the third chunk, not the hundredth
|
||||
|
||||
|
||||
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
clock = iter(range(0, 1000, 10))
|
||||
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
|
||||
monkeypatch.setattr(ats, "DEADLINE", 25.0)
|
||||
body = (b"x" for _ in range(100))
|
||||
with _client(lambda request: httpx.Response(200, content=body)) as client, \
|
||||
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
|
||||
|
||||
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/v0/postings/old":
|
||||
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
|
||||
if request.url.path == "/v0/postings/away":
|
||||
return httpx.Response(302, headers={"location": "https://evil.example/"})
|
||||
return httpx.Response(200, content=b"[]")
|
||||
|
||||
with _client(handler) as client:
|
||||
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
|
||||
with pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
|
||||
|
||||
|
||||
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
|
||||
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
|
||||
429 on a later page still reaches it as an `HTTPStatusError`."""
|
||||
bodies: list[Any] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
payload = json.loads(request.content or b"{}")
|
||||
bodies.append(payload)
|
||||
if payload.get("token") == "stop":
|
||||
return httpx.Response(429)
|
||||
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
|
||||
if payload:
|
||||
page["total"], page["nextPage"] = 4, "stop"
|
||||
return httpx.Response(200, json=page)
|
||||
|
||||
with _client(handler) as client:
|
||||
rows, complete = ats._fetch_rows(client, "workable:exotec")
|
||||
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
|
||||
assert (len(rows), complete) == (3, False)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- paging
|
||||
|
||||
def _page(name: str) -> bytes:
|
||||
|
||||
Reference in new issue
Block a user