Files
jobboard/tests/test_storage.py
T
WolfandClaude Fable 5.1 9035529e0d Summarise and fetch text: a summary stage after score, two role routes
app/jobs/summarise.py is a ModelTask beside extract: about 60 words per role
from title, company, place and the fetched page else the description (capped
at 8000 characters). The pull cascade gains a summarise stage after score that
takes only roles without a summary in the top summarise.per_agent of an
enabled agent's ungated scores, and finds nothing when summarise.enabled is
false or no model is reachable. A reply that leaves a role unanswered is
counted in the state table and the role is given up after three.

POST /api/roles/{uid}/text scrapes original_url once and stores
article_text; POST /api/roles/{uid}/summarise queues a summarise_batch for
that uid alone, deduped per role.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-25 21:39:43 +02:00

358 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""The store: schema, role upserts, closure, companies, subscriptions, the tracker."""
from __future__ import annotations
import sqlite3
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
from pydantic import ValidationError
from desk.models import Item
from app.models import Application, company_key, place_key, title_key
from app.storage import LATEST, Store
class Clock:
def __init__(self) -> None:
self.at = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
def __call__(self) -> datetime:
return self.at
def tick(self, hours: float = 1) -> None:
self.at += timedelta(hours=hours)
@pytest.fixture
def clock() -> Clock:
return Clock()
@pytest.fixture
def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
s = Store(tmp_path / "roledesk.db")
s.clock = clock
yield s
def posting(source_id: str = "123", **raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/acme/{source_id}",
title="Senior Backend Engineer (m/w/d)", body="We build things.",
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
item.source = "ats"
return item
# ------------------------------------------------------------------ schema
def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
path = tmp_path / "new.db"
store = Store(path)
with sqlite3.connect(path) as conn:
tables = {r[0] for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
assert conn.execute("PRAGMA user_version").fetchone()[0] == LATEST
assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
indexes = {r[0] for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='index'")}
assert {"roles", "companies", "subscriptions", "applications",
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
def test_a_newer_file_is_refused(tmp_path: Path) -> None:
path = tmp_path / "future.db"
Store(path)
with sqlite3.connect(path) as conn:
conn.execute(f"PRAGMA user_version = {LATEST + 1}")
with pytest.raises(RuntimeError, match=r"Upgrade roledesk"):
Store(path)
def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
with sqlite3.connect(path) as conn:
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
fresh, old = tmp_path / "fresh.db", tmp_path / "old.db"
Store(fresh)
Store(old)
with sqlite3.connect(old) as conn:
for column in ("permit", "languages", "extract_tries", "vote", "voted_at",
"summary", "summarised_at", "article_text", "article_at"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
for column in ("intent_key", "intent_vec"):
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
conn.execute("DROP TABLE vectors")
conn.execute("PRAGMA user_version = 1")
assert Store(old).migrations_applied == list(range(2, LATEST + 1))
# ALTER appends, so the order differs; every read is by column name.
for table in ("roles", "agents", "vectors"):
assert sorted(_columns(old, table)) == sorted(_columns(fresh, table))
def test_state_is_a_mapping(store: Store) -> None:
store.state["ats:last"] = "2026-09-23"
assert store.state["ats:last"] == "2026-09-23"
assert dict(store.state) == {"ats:last": "2026-09-23"}
del store.state["ats:last"]
assert "ats:last" not in store.state
# ------------------------------------------------------------------- keys
def test_keys() -> None:
assert title_key("Senior Backend Engineer (m/w/d)") == "senior backend engineer"
assert title_key("Data Scientist (f/m/x) – Berlin") == "data scientist berlin" # noqa: RUF001
assert title_key("Product Designer m/w/d") == "product designer"
assert title_key("Engineer (all genders)") == "engineer"
assert place_key("Zürich, Switzerland") == "zurich switzerland"
assert company_key("Trade Republic Bank GmbH") == "trade-republic-bank"
assert company_key("Celonis SE") == company_key("celonis") == "celonis"
# ------------------------------------------------------------------- roles
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
store: Store, clock: Clock) -> None:
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is True
first = store.role(posting().uid)
assert first is not None
clock.tick()
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is False
again = store.role(posting().uid)
assert again is not None
assert again.first_seen == first.first_seen == first.created_at
assert again.last_seen > first.last_seen
assert (again.company_key, again.title_key, again.place_key) == \
("acme", "senior backend engineer", "berlin")
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
salary_max=95000, currency="EUR", department="Eng"))
role = store.role(posting().uid)
assert role is not None
assert (role.remote, role.salary_min, role.salary_max, role.currency) == \
("remote", 80000, 95000, "EUR")
assert role.raw["department"] == "Eng"
assert role.description == "We build things."
assert role.original_url.endswith("/123")
# A later fetch saying less does not erase what was known.
store.upsert_role(posting())
role = store.role(posting().uid)
assert role is not None
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
def test_company_falls_back_to_the_subscription(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
store.upsert_role(posting(), sub)
role = store.role(posting().uid)
assert role is not None
assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id)
def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin"))
clock.tick()
store.upsert_role(posting("2", company="Acme", place="Munich"))
clock.tick()
store.upsert_role(posting("3", company="Other", place="Berlin"))
assert [r.source_id for r in store.roles()] == ["3", "2", "1"]
assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"]
assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"]
assert [r.source_id for r in store.roles(limit=1)] == ["3"]
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin",
function="engineering", remote=True))
clock.tick(hours=48)
store.upsert_role(posting("2", company="Acme", place="Munich",
function="sales"))
assert [r.source_id for r in store.roles(q="senior engineer")] == ["2", "1"]
assert [r.source_id for r in store.roles(q="backend nonsense")] == []
assert [r.source_id for r in store.roles(function="engineering")] == ["1"]
assert [r.source_id for r in store.roles(remote="remote")] == ["1"]
assert [r.source_id for r in store.roles(company="ac")] == ["2", "1"]
assert [r.source_id for r in store.roles(place="berlin")] == ["1"]
assert [r.source_id for r in store.roles(days=1)] == ["2"]
assert [r.source_id for r in store.roles(days=30)] == ["2", "1"]
def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None:
store.add_company("Acme", key="acme", affinity=-1)
store.upsert_role(posting("other", company="Other"))
clock.tick()
for i in range(3):
store.upsert_role(posting(f"a{i}", company="Acme"))
clock.tick()
# Acme's roles are the three newest, so a plain limit=1 shows only Acme;
# hide_muted has to reach past all three to find "other" rather than
# just dropping Acme's rows out of an already-limited page.
assert [r.source_id for r in store.roles(limit=1)] == ["a2"]
assert [r.source_id for r in store.roles(limit=1, hide_muted=True)] == ["other"]
assert {r.source_id for r in store.roles(hide_muted=True)} == {"other"}
assert {r.source_id for r in store.roles()} == {"a0", "a1", "a2", "other"}
def test_mark_closed_closes_what_the_board_no_longer_lists(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme")
other = store.add_subscription("ats", "greenhouse:other")
for source_id in ("1", "2", "3"):
store.upsert_role(posting(source_id), sub)
store.upsert_role(posting("9"), other)
assert store.mark_closed("ats", ["1", "3"], sub.id) == 1
assert {r.source_id for r in store.roles()} == {"1", "3", "9"}
assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"}
# Listed again: open again.
store.upsert_role(posting("2"), sub)
assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"}
def test_an_item_without_a_source_is_refused(store: Store) -> None:
item = posting()
item.source = ""
with pytest.raises(ValueError, match="no source"):
store.upsert_role(item)
# --------------------------------------------------------------- companies
def test_companies_crud(store: Store) -> None:
acme = store.add_company("Acme GmbH", domain="acme.example")
assert acme.key == "acme" and acme.affinity == 0 and acme.status == "researching"
with pytest.raises(ValueError, match="already exists"):
store.add_company("Acme")
updated = store.update_company("acme", affinity=2, status="contacted")
assert (updated.affinity, updated.status) == (2, "contacted")
with pytest.raises(ValueError, match="affinity"):
store.update_company("acme", affinity=5)
with pytest.raises(ValueError, match="not a company field"):
store.update_company("acme", created_at="x")
assert [c.key for c in store.companies()] == ["acme"]
assert store.remove_company("acme") is True
assert store.company("acme") is None
def test_open_role_counts_and_company_boards(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme")
store.upsert_role(posting("1", company="Acme", place="Berlin"), sub)
store.upsert_role(posting("2", company="Acme", place="Munich"), sub)
store.mark_closed("ats", ["1"], sub.id)
assert store.open_role_counts() == {"acme": 1}
assert store.company_boards() == {"acme": "greenhouse"}
# ----------------------------------------------------------- subscriptions
def test_subscriptions_crud(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
company="Acme", board="Greenhouse", params={"q": 1})
assert (sub.company, sub.board, sub.params, sub.fetch) == \
("Acme", "Greenhouse", {"q": 1}, 1)
with pytest.raises(ValueError):
store.add_subscription("ats", "greenhouse:acme")
assert store.subscription_for("ats", "Acme") == sub
store.update_subscription(sub.id, fetch=False)
assert store.subscriptions(fetchable=True) == []
assert [s.id for s in store.subscriptions(source="ats")] == [sub.id]
gone = store.unsubscribe(sub.id)
assert (gone.subscribed, gone.fetch, gone.show) == (0, 0, 0)
# ------------------------------------------------------------ applications
def test_a_status_change_writes_an_event(store: Store, clock: Clock) -> None:
saved = store.save_application(Application(company_key="acme", role_title="Engineer"))
assert saved.id and saved.status == "saved"
clock.tick()
store.set_application_status(saved.id, "applied")
clock.tick()
store.save_application(saved.model_copy(update={"status": "applied", "notes": "x"}))
clock.tick()
store.set_application_status(saved.id, "screen")
events = store.application_events(saved.id)
assert [e.status for e in events] == ["saved", "applied", "screen"]
assert events[0].at < events[1].at < events[2].at
assert [a.notes for a in store.applications()] == ["x"]
def test_an_invalid_status_is_refused(store: Store) -> None:
saved = store.save_application(Application(role_title="Engineer"))
with pytest.raises(ValueError, match="not an application status"):
store.set_application_status(saved.id, "ghosted")
with pytest.raises(ValidationError):
Application(status="ghosted")
with pytest.raises(sqlite3.IntegrityError), store.transaction() as conn:
conn.execute("UPDATE applications SET status='ghosted' WHERE id=?", (saved.id,))
assert [e.status for e in store.application_events(saved.id)] == ["saved"]
def test_events_are_append_only(store: Store) -> None:
saved = store.save_application(Application(role_title="Engineer"))
with pytest.raises(sqlite3.IntegrityError, match="append-only"), \
store.transaction() as conn:
conn.execute("UPDATE application_events SET status='offer'")
assert store.application_events(saved.id)[0].status == "saved"
def test_an_application_may_point_at_a_role(store: Store) -> None:
store.upsert_role(posting())
saved = store.save_application(Application(role_uid=posting().uid, status="applied"))
assert saved.role_uid == posting().uid
assert [e.status for e in store.application_events(saved.id)] == ["applied"]
# -------------------------------------------------- summary and posting text
def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
store.upsert_role(posting())
role = store.set_article_text(posting().uid, "The whole posting.")
assert role is not None
assert role.article_text == "The whole posting." and role.article_at == store.now()
assert store.set_article_text("nope", "x") is None
def test_a_summary_is_written_once(store: Store) -> None:
store.upsert_role(posting())
uid = posting().uid
assert store.set_summary(uid, "First.") is True
assert store.set_summary(uid, "Second.") is False
role = store.role(uid)
assert role is not None and role.summary == "First." and role.summarised_at
assert store.set_summary("nope", "x") is False
def test_summarise_waits_on_the_top_of_each_enabled_agent(store: Store) -> None:
"""Two agents each contribute their own top one; a closed role, a hidden
company's role and a gated score are not in anybody's top."""
for sid, company in (("a", "Acme"), ("b", "Acme"), ("c", "Hidden"), ("d", "Acme")):
store.upsert_role(posting(sid, company=company))
a, b, c, d = (posting(s).uid for s in "abcd")
store.add_company("Hidden", key="hidden", affinity=-1)
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
one = store.save_agent("One")
two = store.save_agent("Two")
store.replace_scores(one.id, [(d, 0.99, ""), (a, 0.9, ""), (b, 0.5, "")])
store.replace_scores(two.id, [(a, 0.95, "remote"), (c, 0.9, ""), (b, 0.8, "")])
waiting = {r.uid for r in store.summarise_waiting(10, per_agent=1)}
# `c` is agent two's best open ungated role but its company is hidden.
assert waiting == {a}
assert {r.uid for r in store.summarise_waiting(10, per_agent=2)} == {a, b}
assert store.summarise_pending(per_agent=2) == 2
assert [r.uid for r in store.summarise_waiting(10, 1, uids=[d])] == [d]