Merge branch 'r2-backend'

# Conflicts:
#	app/storage/schema.py
This commit is contained in:
Wolf committed 2026-09-29 15:22:03 +02:00
commit 562fa3ab09
15 files changed
+924 -157

No files matched your search

+67
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import json
import sys
from pathlib import Path
from collections.abc import Callable, Iterator
from typing import Any
import httpx
@@ -333,6 +334,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
return httpx.Client(transport=httpx.MockTransport(handler))
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
monkeypatch: pytest.MonkeyPatch) -> None:
sent: list[int] = []
def chunks() -> Iterator[bytes]:
for i in range(100):
sent.append(i)
yield b"x" * 10
monkeypatch.setattr(ats, "MAX_BYTES", 25)
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
pytest.raises(ats._TooLarge):
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
assert len(sent) == 3 # refused on the third chunk, not the hundredth
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
monkeypatch: pytest.MonkeyPatch) -> None:
clock = iter(range(0, 1000, 10))
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
monkeypatch.setattr(ats, "DEADLINE", 25.0)
body = (b"x" for _ in range(100))
with _client(lambda request: httpx.Response(200, content=body)) as client, \
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
def handler(request: httpx.Request) -> httpx.Response:
if request.url.path == "/v0/postings/old":
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
if request.url.path == "/v0/postings/away":
return httpx.Response(302, headers={"location": "https://evil.example/"})
return httpx.Response(200, content=b"[]")
with _client(handler) as client:
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
with pytest.raises(ats._Refused):
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
429 on a later page still reaches it as an `HTTPStatusError`."""
bodies: list[Any] = []
def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content or b"{}")
bodies.append(payload)
if payload.get("token") == "stop":
return httpx.Response(429)
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
if payload:
page["total"], page["nextPage"] = 4, "stop"
return httpx.Response(200, json=page)
with _client(handler) as client:
rows, complete = ats._fetch_rows(client, "workable:exotec")
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
assert (len(rows), complete) == (3, False)
# ----------------------------------------------------------------- paging
def _page(name: str) -> bytes:
+27
View File
@@ -50,6 +50,33 @@ def store(tmp_path: Path) -> Store:
("London or New York", ""),
("Remote", ""),
("", ""),
("Wrocław", "PL"), # ł and đ do not decompose under NFKD
("Łódź, Poland", "PL"),
("Bogota, CO", "CO"), # a state code that is a country's too
("Paderborn, DE", ""), # ... is no US state without a US place
("Sliema, MT", ""),
("Casablanca, MA", "MA"),
("Cambridge, MA", "US"),
("Chicago, IL", "US"),
("Madrid, MD, Spain", "ES"),
("Dublin, London", "EU"), # several European countries
("Anywhere in France, Belgium, Spain", "EU"),
("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"),
("Remote, EU or US", ""), # a region and a country outside it
("London, Ontario", "CA"), # a province beats the city before it
("Bellevue, Washington", "US"),
("Tilburg, NB, Netherlands", "NL"), # a country beats a province code
("Atlanta, Georgia", "US"),
("Tbilisi, Georgia", "GE"),
("Malaysia", "MY"),
("Belgrade, Serbia", "RS"),
("Dubai", "AE"),
("United Arab Emirates", "AE"),
("Buenos Aires, , Argentina", "AR"),
("Leicester (GB)", "GB"), # a code in parentheses when nothing else
("Cagliari (CA)", "IT"), # ... and not over a name
("Hybrid", ""),
("Distributed", ""),
])
def test_country_of(place: str, code: str) -> None:
assert country_of(place) == code
+28 -1
View File
@@ -66,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 12
assert LATEST == 13
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -260,6 +260,33 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen
def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
tmp_path: Path, clock: Clock) -> None:
path = tmp_path / "v11.db"
store = Store(path)
store.clock = clock
wrong = posting("1", place="Bogota, CO") # was read as US
several = posting("2", place="Dublin, London") # was ''
given = posting("3", place="Bogota, CO", country="US") # the board said so
extracted = posting("4", place="Bogota, CO")
for item in (wrong, several, given, extracted):
store.upsert_role(item)
with sqlite3.connect(path) as conn:
conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'")
conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,))
conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?",
(extracted.uid,))
conn.execute("PRAGMA user_version = 12")
migrated = Store(path)
assert migrated.migrations_applied == list(range(13, LATEST + 1))
countries = {uid: role.country for uid in (wrong.uid, several.uid, given.uid, extracted.uid)
if (role := migrated.role(uid)) is not None}
assert countries == {wrong.uid: "CO", several.uid: "EU", given.uid: "US",
extracted.uid: "US"}
def test_state_is_a_mapping(store: Store) -> None:
store.state["ats:last"] = "2026-09-23"
assert store.state["ats:last"] == "2026-09-23"
+82 -3
View File
@@ -5,6 +5,7 @@ worker beyond starting and stopping it with the app."""
from __future__ import annotations
import json
import sqlite3
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -18,6 +19,7 @@ from app.config import Config, ServerConfig
from app.main import create_app
from app.storage import Store
from app.web import MOVED, SETTINGS, TABS
from app.web.api import _score_after_change
from app.web.pages import SECTIONS
from auth import AuthConfig
from desk.models import Item
@@ -411,7 +413,7 @@ def test_companies_lists_board_and_open_role_count(
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme", ats="greenhouse", country="DK")
store.upsert_role(posting("1", company="Acme"), sub)
store.add_company("Other", key="other") # no board found
store.add_company("Other", key="other", domain="other.example") # no board found
store.add_company("Novo", key="novo", ats="successfactors", # followed, not readable
careers_url="https://novo.example/careers")
@@ -421,13 +423,47 @@ def test_companies_lists_board_and_open_role_count(
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
"ats": "greenhouse", "ats_slug": "", "careers_url": "",
"country": "DK", "failing_since": "", "failing_reason": ""}
"country": "DK", "failing_since": "", "failing_reason": "",
"from_search": False}
[other] = [c for c in res.json() if c["key"] == "other"]
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
"", None, None, "")
[novo] = [c for c in res.json() if c["key"] == "novo"]
assert (novo["board"], novo["subscription_id"], novo["ats"], novo["careers_url"]) == (
"", None, "successfactors", "https://novo.example/careers")
assert not any(c["from_search"] for c in res.json())
def test_judging_a_company_known_only_from_a_search_creates_its_row(
client: TestClient, store: Store) -> None:
"""Jobindex and Adzuna employers have roles and no company row. The first
affinity change makes a minimal one, named by its roles, which the
Companies list marks `from_search`, unlike a followed company without a
board, and a company whose board was detached."""
store.save_agent("A")
search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex")
for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")):
item = posting(f"n{i}", company=name)
item.source = "search"
store.upsert_role(item, search)
key = "novo-nordisk"
assert store.company(key) is None
res = client.post(f"/api/companies/{key}/affinity", json={"affinity": -1})
assert res.status_code == 200, res.text
assert (res.json()["name"], res.json()["affinity"]) == ("Novo Nordisk A/S", -1)
assert client.post("/api/companies/nobody/affinity", json={"affinity": 1}).status_code == 404
store.add_company("Pleo", key="pleo", domain="pleo.io") # followed, no board found
sub = store.add_subscription("ats", "lever:gone", company="Gone", board="lever")
store.add_company("Gone", key="gone")
store.set_company_board("gone", "") # detached
assert not store.subscription(sub.id).show # type: ignore[union-attr]
rows = {c["key"]: c for c in client.get("/api/companies").json()}
assert rows["gone"]["board"] == ""
assert rows[key]["from_search"] and rows[key]["open_roles"] == 3
assert (rows[key]["board"], rows[key]["ats"], rows[key]["careers_url"]) == ("", "", "")
assert not rows["pleo"]["from_search"] and not rows["gone"]["from_search"]
def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,
@@ -468,7 +504,7 @@ def test_set_affinity_validates_updates_and_enqueues_a_score(
assert res.json()["affinity"] == 2
assert store.company("acme", agent.id).affinity == 2 # type: ignore[union-attr]
assert store.company("acme", other.id).affinity == 0 # type: ignore[union-attr]
assert queue.jobs.last("score", states=("queued", "running", "done")) is not None
assert queue.jobs.get(res.json()["score_job"]).kind == "score"
res = client.post("/api/companies/acme/affinity", json={"affinity": -1, "agent": other.id})
assert res.status_code == 200 and res.json()["affinity"] == -1
@@ -482,6 +518,49 @@ def test_set_affinity_validates_updates_and_enqueues_a_score(
assert client.get("/api/companies", params={"agent": 999}).status_code == 404
def test_a_score_asked_for_during_a_run_is_one_follow_up_after_it(queue: Any) -> None:
"""No worker runs here (no client), so the test moves the jobs itself."""
first = _score_after_change(queue)
assert first is not None and first.dedup_key == "score"
assert _score_after_change(queue).id == first.id # queued: it will read the change
assert queue.jobs.claim({"cpu": 1}).id == first.id
during = {_score_after_change(queue).id for _ in range(10)}
assert len(during) == 1 and first.id not in during
follow = queue.jobs.get(during.pop())
assert (follow.depends_on, follow.after, follow.dedup_key) == (
[first.id], "done", f"score:after:{first.id}")
queue.jobs.finish(first.id, "done")
assert _score_after_change(queue).id == follow.id # queued, not started: still the one
assert queue.jobs.claim({"cpu": 1}).id == follow.id
after = _score_after_change(queue)
assert after.id not in (first.id, follow.id) and after.dedup_key == "score"
def test_score_after_change_retries_the_queues_dedup_race(
queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
"""`Jobs.enqueue` re-raises the UNIQUE failure when the live job it
collided with finishes before it reads it back."""
real, calls = queue.enqueue, []
def racy(*args: Any, **kwargs: Any) -> Any:
calls.append(args)
if len(calls) == 1:
raise sqlite3.IntegrityError("UNIQUE constraint failed: jobs.dedup_key")
return real(*args, **kwargs)
monkeypatch.setattr(queue, "enqueue", racy)
assert _score_after_change(queue) is not None and len(calls) == 2
def test_post_score_answers_with_the_job_to_wait_for(client: TestClient, queue: Any) -> None:
res = client.post("/api/score")
assert res.status_code == 200
assert queue.jobs.get(res.json()["id"]).kind == "score"
# ----------------------------------------------------------- applications
def test_save_creates_an_application_with_a_saved_event(