Merge branch 'r2-backend'
# Conflicts: # app/storage/schema.py
This commit is contained in:
commit
562fa3ab09
15 files changed
+924
-157
No files matched your search
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -333,6 +334,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
sent: list[int] = []
|
||||
|
||||
def chunks() -> Iterator[bytes]:
|
||||
for i in range(100):
|
||||
sent.append(i)
|
||||
yield b"x" * 10
|
||||
|
||||
monkeypatch.setattr(ats, "MAX_BYTES", 25)
|
||||
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
|
||||
pytest.raises(ats._TooLarge):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
assert len(sent) == 3 # refused on the third chunk, not the hundredth
|
||||
|
||||
|
||||
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
clock = iter(range(0, 1000, 10))
|
||||
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
|
||||
monkeypatch.setattr(ats, "DEADLINE", 25.0)
|
||||
body = (b"x" for _ in range(100))
|
||||
with _client(lambda request: httpx.Response(200, content=body)) as client, \
|
||||
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
|
||||
|
||||
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/v0/postings/old":
|
||||
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
|
||||
if request.url.path == "/v0/postings/away":
|
||||
return httpx.Response(302, headers={"location": "https://evil.example/"})
|
||||
return httpx.Response(200, content=b"[]")
|
||||
|
||||
with _client(handler) as client:
|
||||
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
|
||||
with pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
|
||||
|
||||
|
||||
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
|
||||
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
|
||||
429 on a later page still reaches it as an `HTTPStatusError`."""
|
||||
bodies: list[Any] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
payload = json.loads(request.content or b"{}")
|
||||
bodies.append(payload)
|
||||
if payload.get("token") == "stop":
|
||||
return httpx.Response(429)
|
||||
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
|
||||
if payload:
|
||||
page["total"], page["nextPage"] = 4, "stop"
|
||||
return httpx.Response(200, json=page)
|
||||
|
||||
with _client(handler) as client:
|
||||
rows, complete = ats._fetch_rows(client, "workable:exotec")
|
||||
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
|
||||
assert (len(rows), complete) == (3, False)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- paging
|
||||
|
||||
def _page(name: str) -> bytes:
|
||||
|
||||
@@ -50,6 +50,33 @@ def store(tmp_path: Path) -> Store:
|
||||
("London or New York", ""),
|
||||
("Remote", ""),
|
||||
("", ""),
|
||||
("Wrocław", "PL"), # ł and đ do not decompose under NFKD
|
||||
("Łódź, Poland", "PL"),
|
||||
("Bogota, CO", "CO"), # a state code that is a country's too
|
||||
("Paderborn, DE", ""), # ... is no US state without a US place
|
||||
("Sliema, MT", ""),
|
||||
("Casablanca, MA", "MA"),
|
||||
("Cambridge, MA", "US"),
|
||||
("Chicago, IL", "US"),
|
||||
("Madrid, MD, Spain", "ES"),
|
||||
("Dublin, London", "EU"), # several European countries
|
||||
("Anywhere in France, Belgium, Spain", "EU"),
|
||||
("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"),
|
||||
("Remote, EU or US", ""), # a region and a country outside it
|
||||
("London, Ontario", "CA"), # a province beats the city before it
|
||||
("Bellevue, Washington", "US"),
|
||||
("Tilburg, NB, Netherlands", "NL"), # a country beats a province code
|
||||
("Atlanta, Georgia", "US"),
|
||||
("Tbilisi, Georgia", "GE"),
|
||||
("Malaysia", "MY"),
|
||||
("Belgrade, Serbia", "RS"),
|
||||
("Dubai", "AE"),
|
||||
("United Arab Emirates", "AE"),
|
||||
("Buenos Aires, , Argentina", "AR"),
|
||||
("Leicester (GB)", "GB"), # a code in parentheses when nothing else
|
||||
("Cagliari (CA)", "IT"), # ... and not over a name
|
||||
("Hybrid", ""),
|
||||
("Distributed", ""),
|
||||
])
|
||||
def test_country_of(place: str, code: str) -> None:
|
||||
assert country_of(place) == code
|
||||
|
||||
+28
-1
@@ -66,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 12
|
||||
assert LATEST == 13
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -260,6 +260,33 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
|
||||
assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen
|
||||
|
||||
|
||||
def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
|
||||
tmp_path: Path, clock: Clock) -> None:
|
||||
path = tmp_path / "v11.db"
|
||||
store = Store(path)
|
||||
store.clock = clock
|
||||
wrong = posting("1", place="Bogota, CO") # was read as US
|
||||
several = posting("2", place="Dublin, London") # was ''
|
||||
given = posting("3", place="Bogota, CO", country="US") # the board said so
|
||||
extracted = posting("4", place="Bogota, CO")
|
||||
for item in (wrong, several, given, extracted):
|
||||
store.upsert_role(item)
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'")
|
||||
conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,))
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?",
|
||||
(extracted.uid,))
|
||||
conn.execute("PRAGMA user_version = 12")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == list(range(13, LATEST + 1))
|
||||
countries = {uid: role.country for uid in (wrong.uid, several.uid, given.uid, extracted.uid)
|
||||
if (role := migrated.role(uid)) is not None}
|
||||
assert countries == {wrong.uid: "CO", several.uid: "EU", given.uid: "US",
|
||||
extracted.uid: "US"}
|
||||
|
||||
|
||||
def test_state_is_a_mapping(store: Store) -> None:
|
||||
store.state["ats:last"] = "2026-09-23"
|
||||
assert store.state["ats:last"] == "2026-09-23"
|
||||
|
||||
+82
-3
@@ -5,6 +5,7 @@ worker beyond starting and stopping it with the app."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -18,6 +19,7 @@ from app.config import Config, ServerConfig
|
||||
from app.main import create_app
|
||||
from app.storage import Store
|
||||
from app.web import MOVED, SETTINGS, TABS
|
||||
from app.web.api import _score_after_change
|
||||
from app.web.pages import SECTIONS
|
||||
from auth import AuthConfig
|
||||
from desk.models import Item
|
||||
@@ -411,7 +413,7 @@ def test_companies_lists_board_and_open_role_count(
|
||||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
||||
store.add_company("Acme", key="acme", ats="greenhouse", country="DK")
|
||||
store.upsert_role(posting("1", company="Acme"), sub)
|
||||
store.add_company("Other", key="other") # no board found
|
||||
store.add_company("Other", key="other", domain="other.example") # no board found
|
||||
store.add_company("Novo", key="novo", ats="successfactors", # followed, not readable
|
||||
careers_url="https://novo.example/careers")
|
||||
|
||||
@@ -421,13 +423,47 @@ def test_companies_lists_board_and_open_role_count(
|
||||
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
|
||||
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
|
||||
"ats": "greenhouse", "ats_slug": "", "careers_url": "",
|
||||
"country": "DK", "failing_since": "", "failing_reason": ""}
|
||||
"country": "DK", "failing_since": "", "failing_reason": "",
|
||||
"from_search": False}
|
||||
[other] = [c for c in res.json() if c["key"] == "other"]
|
||||
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
|
||||
"", None, None, "")
|
||||
[novo] = [c for c in res.json() if c["key"] == "novo"]
|
||||
assert (novo["board"], novo["subscription_id"], novo["ats"], novo["careers_url"]) == (
|
||||
"", None, "successfactors", "https://novo.example/careers")
|
||||
assert not any(c["from_search"] for c in res.json())
|
||||
|
||||
|
||||
def test_judging_a_company_known_only_from_a_search_creates_its_row(
|
||||
client: TestClient, store: Store) -> None:
|
||||
"""Jobindex and Adzuna employers have roles and no company row. The first
|
||||
affinity change makes a minimal one, named by its roles, which the
|
||||
Companies list marks `from_search`, unlike a followed company without a
|
||||
board, and a company whose board was detached."""
|
||||
store.save_agent("A")
|
||||
search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex")
|
||||
for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")):
|
||||
item = posting(f"n{i}", company=name)
|
||||
item.source = "search"
|
||||
store.upsert_role(item, search)
|
||||
key = "novo-nordisk"
|
||||
assert store.company(key) is None
|
||||
|
||||
res = client.post(f"/api/companies/{key}/affinity", json={"affinity": -1})
|
||||
assert res.status_code == 200, res.text
|
||||
assert (res.json()["name"], res.json()["affinity"]) == ("Novo Nordisk A/S", -1)
|
||||
assert client.post("/api/companies/nobody/affinity", json={"affinity": 1}).status_code == 404
|
||||
|
||||
store.add_company("Pleo", key="pleo", domain="pleo.io") # followed, no board found
|
||||
sub = store.add_subscription("ats", "lever:gone", company="Gone", board="lever")
|
||||
store.add_company("Gone", key="gone")
|
||||
store.set_company_board("gone", "") # detached
|
||||
assert not store.subscription(sub.id).show # type: ignore[union-attr]
|
||||
rows = {c["key"]: c for c in client.get("/api/companies").json()}
|
||||
assert rows["gone"]["board"] == ""
|
||||
assert rows[key]["from_search"] and rows[key]["open_roles"] == 3
|
||||
assert (rows[key]["board"], rows[key]["ats"], rows[key]["careers_url"]) == ("", "", "")
|
||||
assert not rows["pleo"]["from_search"] and not rows["gone"]["from_search"]
|
||||
|
||||
|
||||
def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,
|
||||
@@ -468,7 +504,7 @@ def test_set_affinity_validates_updates_and_enqueues_a_score(
|
||||
assert res.json()["affinity"] == 2
|
||||
assert store.company("acme", agent.id).affinity == 2 # type: ignore[union-attr]
|
||||
assert store.company("acme", other.id).affinity == 0 # type: ignore[union-attr]
|
||||
assert queue.jobs.last("score", states=("queued", "running", "done")) is not None
|
||||
assert queue.jobs.get(res.json()["score_job"]).kind == "score"
|
||||
|
||||
res = client.post("/api/companies/acme/affinity", json={"affinity": -1, "agent": other.id})
|
||||
assert res.status_code == 200 and res.json()["affinity"] == -1
|
||||
@@ -482,6 +518,49 @@ def test_set_affinity_validates_updates_and_enqueues_a_score(
|
||||
assert client.get("/api/companies", params={"agent": 999}).status_code == 404
|
||||
|
||||
|
||||
def test_a_score_asked_for_during_a_run_is_one_follow_up_after_it(queue: Any) -> None:
|
||||
"""No worker runs here (no client), so the test moves the jobs itself."""
|
||||
first = _score_after_change(queue)
|
||||
assert first is not None and first.dedup_key == "score"
|
||||
assert _score_after_change(queue).id == first.id # queued: it will read the change
|
||||
|
||||
assert queue.jobs.claim({"cpu": 1}).id == first.id
|
||||
during = {_score_after_change(queue).id for _ in range(10)}
|
||||
assert len(during) == 1 and first.id not in during
|
||||
follow = queue.jobs.get(during.pop())
|
||||
assert (follow.depends_on, follow.after, follow.dedup_key) == (
|
||||
[first.id], "done", f"score:after:{first.id}")
|
||||
|
||||
queue.jobs.finish(first.id, "done")
|
||||
assert _score_after_change(queue).id == follow.id # queued, not started: still the one
|
||||
|
||||
assert queue.jobs.claim({"cpu": 1}).id == follow.id
|
||||
after = _score_after_change(queue)
|
||||
assert after.id not in (first.id, follow.id) and after.dedup_key == "score"
|
||||
|
||||
|
||||
def test_score_after_change_retries_the_queues_dedup_race(
|
||||
queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`Jobs.enqueue` re-raises the UNIQUE failure when the live job it
|
||||
collided with finishes before it reads it back."""
|
||||
real, calls = queue.enqueue, []
|
||||
|
||||
def racy(*args: Any, **kwargs: Any) -> Any:
|
||||
calls.append(args)
|
||||
if len(calls) == 1:
|
||||
raise sqlite3.IntegrityError("UNIQUE constraint failed: jobs.dedup_key")
|
||||
return real(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(queue, "enqueue", racy)
|
||||
assert _score_after_change(queue) is not None and len(calls) == 2
|
||||
|
||||
|
||||
def test_post_score_answers_with_the_job_to_wait_for(client: TestClient, queue: Any) -> None:
|
||||
res = client.post("/api/score")
|
||||
assert res.status_code == 200
|
||||
assert queue.jobs.get(res.json()["id"]).kind == "score"
|
||||
|
||||
|
||||
# ----------------------------------------------------------- applications
|
||||
|
||||
def test_save_creates_an_application_with_a_saved_event(
|
||||
|
||||
Reference in new issue
Block a user