1312 lines
61 KiB
Python
1312 lines
61 KiB
Python
"""The store: schema, role upserts, closure, companies, subscriptions, the tracker."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import sqlite3
|
||
from collections.abc import Iterator
|
||
from datetime import UTC, datetime, timedelta
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from pydantic import ValidationError
|
||
|
||
from desk.models import Item
|
||
import numpy as np
|
||
|
||
from app.models import (
|
||
Application, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at,
|
||
posted_host, title_key,
|
||
)
|
||
from app.storage.store import pack
|
||
from app.storage import LATEST, Store
|
||
|
||
|
||
class Clock:
|
||
def __init__(self) -> None:
|
||
self.at = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
||
|
||
def __call__(self) -> datetime:
|
||
return self.at
|
||
|
||
def tick(self, hours: float = 1) -> None:
|
||
self.at += timedelta(hours=hours)
|
||
|
||
|
||
@pytest.fixture
|
||
def clock() -> Clock:
|
||
return Clock()
|
||
|
||
|
||
@pytest.fixture
|
||
def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
|
||
s = Store(tmp_path / "roledesk.db")
|
||
s.clock = clock
|
||
yield s
|
||
|
||
|
||
def posting(source_id: str = "123", *,
|
||
published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: object) -> Item:
|
||
item = Item(source_id=source_id, location=f"https://boards.example/acme/{source_id}",
|
||
title="Senior Backend Engineer (m/w/d)", body="We build things.",
|
||
published_at=published_at, raw=dict(raw))
|
||
item.source = "ats"
|
||
return item
|
||
|
||
|
||
# ------------------------------------------------------------------ schema
|
||
|
||
def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||
path = tmp_path / "new.db"
|
||
store = Store(path)
|
||
with sqlite3.connect(path) as conn:
|
||
tables = {r[0] for r in conn.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table'")}
|
||
assert conn.execute("PRAGMA user_version").fetchone()[0] == LATEST
|
||
assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
|
||
indexes = {r[0] for r in conn.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='index'")}
|
||
assert {"roles", "companies", "subscriptions", "applications",
|
||
"application_events", "agents", "scores", "state"} <= tables
|
||
assert "idx_roles_cluster" in indexes
|
||
assert store.migrations_applied == []
|
||
assert LATEST == 15
|
||
# A second open finds it current and applies nothing.
|
||
assert Store(path).migrations_applied == []
|
||
|
||
|
||
def test_a_newer_file_is_refused(tmp_path: Path) -> None:
|
||
path = tmp_path / "future.db"
|
||
Store(path)
|
||
with sqlite3.connect(path) as conn:
|
||
conn.execute(f"PRAGMA user_version = {LATEST + 1}")
|
||
with pytest.raises(RuntimeError, match=r"Upgrade roledesk"):
|
||
Store(path)
|
||
|
||
|
||
def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
|
||
with sqlite3.connect(path) as conn:
|
||
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
|
||
|
||
|
||
def _undo_v15(conn: sqlite3.Connection) -> None:
|
||
"""A fresh file put back to the v14 shape, without the by-hand columns of
|
||
a company, and then to the v13 one by `_undo_v14`. Every downgrade below
|
||
starts here, since v15 is the last migration."""
|
||
conn.execute("ALTER TABLE companies DROP COLUMN hand_reason")
|
||
conn.execute("ALTER TABLE companies DROP COLUMN checked_at")
|
||
_undo_v14(conn)
|
||
|
||
|
||
def _undo_v14(conn: sqlite3.Connection) -> None:
|
||
"""A fresh file put back to the v13 shape: an agent's intent and its
|
||
cached vector on the agent row, no prompts, no prompt on a score."""
|
||
conn.execute("DROP TABLE agent_prompts")
|
||
conn.execute("ALTER TABLE scores DROP COLUMN prompt")
|
||
conn.execute("ALTER TABLE agents ADD COLUMN intent TEXT NOT NULL DEFAULT ''")
|
||
conn.execute("ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT ''")
|
||
conn.execute("ALTER TABLE agents ADD COLUMN intent_vec BLOB")
|
||
|
||
|
||
def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
|
||
fresh, old = tmp_path / "fresh.db", tmp_path / "old.db"
|
||
Store(fresh)
|
||
Store(old)
|
||
with sqlite3.connect(old) as conn:
|
||
_undo_v15(conn)
|
||
conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
|
||
for column in ("permit", "languages", "extract_tries", "summary", "summarised_at",
|
||
"article_text", "article_at", "read_at", "posted_at", "salary_text"):
|
||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||
for column in ("intent_key", "intent_vec", "limits"): # v3, v8
|
||
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
|
||
# v1 had affinity on the company; the vote columns v4 added and v6
|
||
# drops again are not in a fresh file, so there is nothing to put back.
|
||
conn.execute("ALTER TABLE companies ADD COLUMN affinity INTEGER NOT NULL DEFAULT 0")
|
||
for column in ("ats", "country", "city", "ats_slug"): # v7, v9
|
||
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
||
for table in ("vectors", "role_tags", "agent_companies", "role_votes"):
|
||
conn.execute(f"DROP TABLE {table}")
|
||
conn.execute("PRAGMA user_version = 1")
|
||
|
||
assert Store(old).migrations_applied == list(range(2, LATEST + 1))
|
||
# ALTER appends, so the order differs; every read is by column name.
|
||
for table in ("roles", "agents", "agent_prompts", "scores", "vectors", "companies",
|
||
"role_tags", "agent_companies", "role_votes"):
|
||
assert sorted(_columns(old, table)) == sorted(_columns(fresh, table))
|
||
|
||
|
||
def test_a_v8_file_gains_ats_slug_and_keeps_its_companies(tmp_path: Path) -> None:
|
||
path = tmp_path / "v8.db"
|
||
Store(path).add_company("Acme", ats="teamtailor", country="DK")
|
||
with sqlite3.connect(path) as conn:
|
||
_undo_v15(conn)
|
||
conn.execute("ALTER TABLE companies DROP COLUMN ats_slug")
|
||
conn.execute("DROP INDEX idx_roles_posted") # v10, which follows
|
||
conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
|
||
conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12, which follows
|
||
conn.execute("PRAGMA user_version = 8")
|
||
|
||
migrated = Store(path)
|
||
|
||
assert migrated.migrations_applied == list(range(9, LATEST + 1))
|
||
acme = migrated.company("acme")
|
||
assert acme is not None
|
||
assert (acme.ats, acme.country, acme.ats_slug) == ("teamtailor", "DK", "")
|
||
|
||
|
||
def _downgrade_to_v4(path: Path) -> None:
|
||
"""A fresh file put back to the v4 shape: votes on the role, affinity on
|
||
the company, no read state, no tags, no per-agent tables."""
|
||
with sqlite3.connect(path) as conn:
|
||
_undo_v15(conn)
|
||
conn.execute("ALTER TABLE roles DROP COLUMN read_at")
|
||
conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
|
||
conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9
|
||
conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12
|
||
conn.execute("DROP TABLE role_tags")
|
||
conn.execute("ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1))")
|
||
conn.execute("ALTER TABLE roles ADD COLUMN voted_at TEXT")
|
||
conn.execute("ALTER TABLE companies ADD COLUMN affinity INTEGER NOT NULL DEFAULT 0")
|
||
for column in ("ats", "country", "city", "ats_slug"): # v7, v9
|
||
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
||
conn.execute("ALTER TABLE agents DROP COLUMN limits") # v8
|
||
conn.execute("DROP TABLE agent_companies")
|
||
conn.execute("DROP TABLE role_votes")
|
||
conn.execute("PRAGMA user_version = 4")
|
||
|
||
|
||
def test_v6_moves_the_votes_and_affinities_to_the_default_agent(tmp_path: Path) -> None:
|
||
path = tmp_path / "v4.db"
|
||
store = Store(path)
|
||
store.upsert_role(posting("liked"))
|
||
store.upsert_role(posting("passed"))
|
||
store.upsert_role(posting("none"))
|
||
store.add_company("Acme", key="acme")
|
||
store.add_company("Other", key="other")
|
||
# The disabled agent has the lower id; the enabled one is the default.
|
||
second = store.save_agent("Second", enabled=False)
|
||
first = store.save_agent("First")
|
||
assert first.id > second.id and store.default_agent().id == first.id
|
||
_downgrade_to_v4(path)
|
||
with sqlite3.connect(path) as conn:
|
||
conn.execute("UPDATE roles SET vote = 1, voted_at = 'then' WHERE uid = ?",
|
||
(posting("liked").uid,))
|
||
conn.execute("UPDATE roles SET vote = -1 WHERE uid = ?", (posting("passed").uid,))
|
||
conn.execute("UPDATE companies SET affinity = -1 WHERE key = 'acme'")
|
||
|
||
migrated = Store(path)
|
||
|
||
assert migrated.migrations_applied == list(range(5, LATEST + 1))
|
||
assert [a.name for a in migrated.agents()] == ["First", "Second"] # none added
|
||
assert migrated.votes(first.id) == {posting("liked").uid: 1, posting("passed").uid: -1}
|
||
assert migrated.votes(second.id) == {}
|
||
liked = migrated.role(posting("liked").uid, first.id)
|
||
assert liked is not None and liked.voted_at == "then"
|
||
passed = migrated.role(posting("passed").uid, first.id)
|
||
assert passed is not None and passed.voted_at # stamped when v4 had none
|
||
assert [c.affinity for c in migrated.companies(first.id)] == [-1, 0]
|
||
assert [c.affinity for c in migrated.companies(second.id)] == [0, 0]
|
||
with sqlite3.connect(path) as conn:
|
||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||
assert not {"vote", "voted_at"} & columns
|
||
assert "affinity" not in {r[1] for r in conn.execute("PRAGMA table_info(companies)")}
|
||
|
||
|
||
def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it(
|
||
tmp_path: Path) -> None:
|
||
path = tmp_path / "v4.db"
|
||
store = Store(path)
|
||
store.upsert_role(posting("liked"))
|
||
store.add_company("Acme", key="acme")
|
||
_downgrade_to_v4(path)
|
||
with sqlite3.connect(path) as conn:
|
||
conn.execute("UPDATE companies SET affinity = 2 WHERE key = 'acme'")
|
||
|
||
migrated = Store(path)
|
||
|
||
[agent] = migrated.agents()
|
||
assert (agent.name, agent.enabled, agent.gates) == ("My ranking", True, [])
|
||
assert agent.created_at.endswith("+00:00")
|
||
assert migrated.default_agent() == agent
|
||
assert migrated.company("acme", agent.id).affinity == 2 # type: ignore[union-attr]
|
||
|
||
|
||
def test_v6_makes_no_agent_when_there_is_nothing_to_move(tmp_path: Path) -> None:
|
||
path = tmp_path / "v4.db"
|
||
store = Store(path)
|
||
store.upsert_role(posting("plain"))
|
||
store.add_company("Acme", key="acme")
|
||
_downgrade_to_v4(path)
|
||
|
||
assert Store(path).migrations_applied == list(range(5, LATEST + 1))
|
||
assert Store(path).agents() == []
|
||
|
||
|
||
def test_a_v4_file_migrates_read_state_and_tags(tmp_path: Path) -> None:
|
||
fresh, old = tmp_path / "fresh.db", tmp_path / "old.db"
|
||
Store(fresh)
|
||
Store(old)
|
||
_downgrade_to_v4(old)
|
||
|
||
assert Store(old).migrations_applied == list(range(5, LATEST + 1))
|
||
assert sorted(_columns(old, "roles")) == sorted(_columns(fresh, "roles"))
|
||
with sqlite3.connect(old) as conn:
|
||
tables = {r[0] for r in conn.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table'")}
|
||
assert "role_tags" in tables
|
||
|
||
|
||
def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
|
||
tmp_path: Path, clock: Clock) -> None:
|
||
path = tmp_path / "v9.db"
|
||
store = Store(path)
|
||
store.clock = clock
|
||
old_date = datetime(2020, 1, 1, tzinfo=UTC)
|
||
trusted = posting("1", board="greenhouse", published_at=old_date)
|
||
untrusted = posting("2", published_at=old_date)
|
||
store.upsert_role(trusted)
|
||
store.upsert_role(untrusted)
|
||
before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid)
|
||
assert before_trusted is not None and before_untrusted is not None
|
||
|
||
with sqlite3.connect(path) as conn:
|
||
_undo_v15(conn)
|
||
conn.execute("DROP INDEX idx_roles_posted")
|
||
conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
|
||
conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12
|
||
conn.execute("PRAGMA user_version = 9")
|
||
|
||
migrated = Store(path)
|
||
|
||
assert migrated.migrations_applied == list(range(10, LATEST + 1))
|
||
after_trusted, after_untrusted = migrated.role(trusted.uid), migrated.role(untrusted.uid)
|
||
assert after_trusted is not None and after_untrusted is not None
|
||
# The backfill reproduces exactly what upsert_role would have written.
|
||
assert after_trusted.posted_at == before_trusted.posted_at == "2020-01-01T00:00:00+00:00"
|
||
assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen
|
||
|
||
|
||
def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
|
||
tmp_path: Path, clock: Clock) -> None:
|
||
path = tmp_path / "v11.db"
|
||
store = Store(path)
|
||
store.clock = clock
|
||
wrong = posting("1", place="Bogota, CO") # was read as US
|
||
several = posting("2", place="Dublin, London") # was ''
|
||
given = posting("3", place="Bogota, CO", country="US") # the board said so
|
||
extracted = posting("4", place="Bogota, CO")
|
||
for item in (wrong, several, given, extracted):
|
||
store.upsert_role(item)
|
||
with sqlite3.connect(path) as conn:
|
||
_undo_v15(conn)
|
||
conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'")
|
||
conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,))
|
||
conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?",
|
||
(extracted.uid,))
|
||
conn.execute("PRAGMA user_version = 12")
|
||
|
||
migrated = Store(path)
|
||
|
||
assert migrated.migrations_applied == list(range(13, LATEST + 1))
|
||
countries = {uid: role.country for uid in (wrong.uid, several.uid, given.uid, extracted.uid)
|
||
if (role := migrated.role(uid)) is not None}
|
||
assert countries == {wrong.uid: "CO", several.uid: "EU", given.uid: "US",
|
||
extracted.uid: "US"}
|
||
|
||
|
||
def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: Path) -> None:
|
||
path = tmp_path / "v13.db"
|
||
store = Store(path)
|
||
store.upsert_role(posting("a"))
|
||
uid = posting("a").uid
|
||
backend = store.save_agent("Backend")
|
||
blank = store.save_agent("Blank")
|
||
empty = store.save_agent("Empty")
|
||
store.replace_scores(backend.id, [(uid, 0.7, "")])
|
||
vec = np.asarray([0.6, 0.8], dtype=np.float32)
|
||
with sqlite3.connect(path) as conn:
|
||
_undo_v15(conn)
|
||
conn.execute("UPDATE agents SET intent = 'python backend', intent_key = 'k', "
|
||
"intent_vec = ? WHERE id = ?", (pack(vec), backend.id))
|
||
conn.execute("UPDATE agents SET intent = ' \n ' WHERE id = ?", (blank.id,))
|
||
conn.execute("PRAGMA user_version = 13")
|
||
|
||
migrated = Store(path)
|
||
|
||
assert migrated.migrations_applied == [14, 15]
|
||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||
Prompt(name="Backend", text="python backend")]
|
||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||
assert migrated.agent(empty.id).prompts == [] # type: ignore[union-attr]
|
||
# The key hashes the model and the text, both unchanged: nothing to re-embed.
|
||
cached = migrated.prompt_vector(backend.id, "Backend", "k")
|
||
assert cached is not None and cached.tolist() == pytest.approx(vec.tolist())
|
||
assert migrated.scores(backend.id) == {uid: (0.7, "")}
|
||
assert migrated.role(uid, backend.id).prompt == "" # type: ignore[union-attr]
|
||
columns = {c[0] for c in _columns(path, "agents")}
|
||
assert not {"intent", "intent_key", "intent_vec"} & columns
|
||
|
||
|
||
def test_v15_gives_a_company_no_reason_and_no_check_and_keeps_its_row(tmp_path: Path) -> None:
|
||
fresh, path = tmp_path / "fresh.db", tmp_path / "v14.db"
|
||
Store(fresh)
|
||
Store(path).add_company("Acme", careers_url="https://acme.example/jobs", notes="met them")
|
||
with sqlite3.connect(path) as conn:
|
||
conn.execute("ALTER TABLE companies DROP COLUMN hand_reason")
|
||
conn.execute("ALTER TABLE companies DROP COLUMN checked_at")
|
||
conn.execute("PRAGMA user_version = 14")
|
||
|
||
migrated = Store(path)
|
||
|
||
assert migrated.migrations_applied == [15]
|
||
acme = migrated.company("acme")
|
||
assert acme is not None
|
||
assert (acme.careers_url, acme.notes, acme.hand_reason, acme.checked_at) == (
|
||
"https://acme.example/jobs", "met them", "", "")
|
||
assert sorted(_columns(path, "companies")) == sorted(_columns(fresh, "companies"))
|
||
|
||
|
||
def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None:
|
||
ai = Prompt(name="AI leadership", text="Head of AI")
|
||
fde = Prompt(name="Forward deployed", text="FDE lead")
|
||
agent = store.save_agent("Wolf", prompts=[ai, fde])
|
||
assert store.agent(agent.id).prompts == [ai, fde] # type: ignore[union-attr]
|
||
store.set_prompt_vector(agent.id, "Forward deployed", "k", np.ones(2))
|
||
|
||
# Reordered and renamed, the same text keeps its vector; a new text has none.
|
||
research = Prompt(name="Research", text="AI research")
|
||
renamed = Prompt(name="FDE", text="FDE lead")
|
||
store.save_agent("Wolf", agent_id=agent.id, prompts=[renamed, research])
|
||
assert [p.name for p in store.agents()[0].prompts] == ["FDE", "Research"]
|
||
assert store.prompt_vector(agent.id, "FDE", "k") is not None
|
||
assert store.prompt_vector(agent.id, "Research", "k") is None
|
||
|
||
# A prompt may have no text; its name is what must be there, and unique.
|
||
store.save_agent("Wolf", agent_id=agent.id, prompts=[Prompt(name="Later")])
|
||
assert store.agents()[0].prompts == [Prompt(name="Later", text="")]
|
||
with pytest.raises(ValueError, match="two prompts are named 'fde'"):
|
||
store.save_agent("Wolf", agent_id=agent.id,
|
||
prompts=[renamed, Prompt(name="fde", text="other")])
|
||
with pytest.raises(ValidationError, match="a prompt needs a name"):
|
||
Prompt(name=" ", text="x")
|
||
assert store.agents()[0].prompts == [Prompt(name="Later")]
|
||
|
||
# Without prompts an intent is the one prompt, named after the agent.
|
||
store.save_agent("Wolf", "python backend", agent_id=agent.id)
|
||
assert store.agent(agent.id).prompts == [ # type: ignore[union-attr]
|
||
Prompt(name="Wolf", text="python backend")]
|
||
store.save_agent("Wolf", agent_id=agent.id)
|
||
assert store.agent(agent.id).prompts == [] # type: ignore[union-attr]
|
||
|
||
store.save_agent("Wolf", agent_id=agent.id, prompts=[ai, fde])
|
||
store.delete_agent(agent.id)
|
||
with store.transaction() as conn:
|
||
assert conn.execute("SELECT COUNT(*) FROM agent_prompts").fetchone()[0] == 0
|
||
|
||
|
||
def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage(
|
||
store: Store) -> None:
|
||
store.upsert_role(posting("a"))
|
||
store.upsert_role(posting("b"))
|
||
a, b = posting("a").uid, posting("b").uid
|
||
top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
||
blank = store.save_agent("Blank", limits=top,
|
||
prompts=[Prompt(name="One"), Prompt(name="Two", text=" \n")])
|
||
store.replace_scores(blank.id, [(a, 0.9, "")])
|
||
assert store.extract_waiting(10) == [] and store.summarise_waiting(10) == []
|
||
|
||
# One prompt with text is enough.
|
||
store.save_agent("Blank", agent_id=blank.id, limits=top,
|
||
prompts=[Prompt(name="One"), Prompt(name="Two", text="python")])
|
||
assert [r.uid for r in store.extract_waiting(10)] == [a]
|
||
assert b not in {r.uid for r in store.summarise_waiting(10)}
|
||
|
||
|
||
def test_state_is_a_mapping(store: Store) -> None:
|
||
store.state["ats:last"] = "2026-09-23"
|
||
assert store.state["ats:last"] == "2026-09-23"
|
||
assert dict(store.state) == {"ats:last": "2026-09-23"}
|
||
del store.state["ats:last"]
|
||
assert "ats:last" not in store.state
|
||
|
||
|
||
# ------------------------------------------------------------------- keys
|
||
|
||
def test_keys() -> None:
|
||
assert title_key("Senior Backend Engineer (m/w/d)") == "senior backend engineer"
|
||
assert title_key("Data Scientist (f/m/x) – Berlin") == "data scientist berlin" # noqa: RUF001
|
||
assert title_key("Product Designer m/w/d") == "product designer"
|
||
assert title_key("Engineer (all genders)") == "engineer"
|
||
assert place_key("Zürich, Switzerland") == "zurich switzerland"
|
||
assert company_key("Trade Republic Bank GmbH") == "trade-republic-bank"
|
||
assert company_key("Celonis SE") == company_key("celonis") == "celonis"
|
||
|
||
|
||
# -------------------------------------------------------------- posted_at
|
||
|
||
def test_posted_host_reads_board_then_provider() -> None:
|
||
assert posted_host({"board": "greenhouse", "provider": "jobindex"}) == "greenhouse"
|
||
assert posted_host({"provider": "jobindex"}) == "jobindex"
|
||
assert posted_host({}) == ""
|
||
|
||
|
||
def test_posted_at_trusts_only_a_known_host() -> None:
|
||
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS
|
||
early = "2020-01-01T00:00:00+00:00"
|
||
late = "2026-09-23T09:00:00+00:00"
|
||
# A trusted host's earlier date wins over first_seen...
|
||
assert posted_at(early, late, "greenhouse") == early
|
||
# ...but an untrusted (or unknown) host never moves it off first_seen.
|
||
assert posted_at(early, late, "taleo") == late
|
||
assert posted_at(early, late, "") == late
|
||
|
||
|
||
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
|
||
first_seen = "2026-09-23T09:00:00+00:00"
|
||
future = "2026-09-24T00:00:00+00:00" # after first_seen
|
||
absurd = "2014-12-31T23:59:59+00:00" # before the 2015 floor
|
||
assert posted_at(future, first_seen, "greenhouse") == first_seen
|
||
assert posted_at(absurd, first_seen, "greenhouse") == first_seen
|
||
# The floor itself, and the instant first_seen, both still count.
|
||
assert posted_at("2015-01-01T00:00:00+00:00", first_seen, "greenhouse") == \
|
||
"2015-01-01T00:00:00+00:00"
|
||
assert posted_at(first_seen, first_seen, "greenhouse") == first_seen
|
||
|
||
|
||
# ------------------------------------------------------------------- roles
|
||
|
||
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
|
||
store: Store, clock: Clock) -> None:
|
||
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is True
|
||
first = store.role(posting().uid)
|
||
assert first is not None
|
||
clock.tick()
|
||
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is False
|
||
again = store.role(posting().uid)
|
||
assert again is not None
|
||
assert again.first_seen == first.first_seen == first.created_at
|
||
assert again.last_seen > first.last_seen
|
||
assert (again.company_key, again.title_key, again.place_key) == \
|
||
("acme", "senior backend engineer", "berlin")
|
||
|
||
|
||
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
||
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
|
||
salary_max=95000, currency="EUR", department="Eng"))
|
||
role = store.role(posting().uid)
|
||
assert role is not None
|
||
assert (role.remote, role.salary_min, role.salary_max, role.currency) == \
|
||
("remote", 80000, 95000, "EUR")
|
||
assert role.raw["department"] == "Eng"
|
||
assert role.description == "We build things."
|
||
assert role.original_url.endswith("/123")
|
||
# A later fetch saying less does not erase what was known.
|
||
store.upsert_role(posting())
|
||
role = store.role(posting().uid)
|
||
assert role is not None
|
||
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
|
||
|
||
|
||
def test_upsert_replaces_salary_text_including_with_empty(store: Store) -> None:
|
||
"""`salary_text` behaves like `title` or `published_at`, not like the
|
||
rest of `_RAW_TEXT`: a board's own pay line replaces what was stored on
|
||
every pull, and a pull that states no pay at all clears it rather than
|
||
leaving a stale line behind — the opposite of the keep-filled rule
|
||
`test_upsert_unpacks_raw_and_keeps_filled_columns` checks just above."""
|
||
store.upsert_role(posting(salary_text="EUR 70,000 to 90,000 per year"))
|
||
role = store.role(posting().uid)
|
||
assert role is not None and role.salary_text == "EUR 70,000 to 90,000 per year"
|
||
|
||
store.upsert_role(posting(salary_text="EUR 75,000 to 95,000 per year"))
|
||
role = store.role(posting().uid)
|
||
assert role is not None and role.salary_text == "EUR 75,000 to 95,000 per year"
|
||
|
||
# The board stops stating pay: the stored line is cleared, not kept.
|
||
store.upsert_role(posting())
|
||
role = store.role(posting().uid)
|
||
assert role is not None and role.salary_text == ""
|
||
|
||
|
||
def test_salary_state_is_derived_from_the_columns_and_the_text(store: Store) -> None:
|
||
"""Not a stored column (docs/decisions.md): 'normalised' needs only
|
||
`salary_min` or `salary_max`, whatever `salary_text` says beside it;
|
||
'stated' is `salary_text` alone; neither is ''."""
|
||
store.upsert_role(posting("1", salary_text="EUR 70,000 to 90,000 per year",
|
||
salary_min=70000, salary_max=90000, currency="EUR"))
|
||
store.upsert_role(posting("2", salary_text="USD 45 to 60 per hour"))
|
||
store.upsert_role(posting("3"))
|
||
by_id = {r.source_id: r for r in store.roles()}
|
||
assert by_id["1"].salary_state == "normalised"
|
||
assert by_id["2"].salary_state == "stated"
|
||
assert by_id["3"].salary_state == ""
|
||
|
||
|
||
def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest(
|
||
store: Store, clock: Clock) -> None:
|
||
"""A board's first pull can carry a stale, too-recent published_at (a
|
||
board that has not backfilled first_published yet, say); a later pull
|
||
that finally reports the true, earlier date must still win — an edit on
|
||
the board only ever moves published_at later, never earlier, so the
|
||
earliest posted_at ever computed is the one to keep."""
|
||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20,
|
||
tzinfo=UTC)))
|
||
first = store.role(posting().uid)
|
||
assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00"
|
||
|
||
clock.tick()
|
||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1,
|
||
tzinfo=UTC)))
|
||
earlier = store.role(posting().uid)
|
||
assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00"
|
||
|
||
clock.tick()
|
||
# A later, LATER published_at (an edit) must not push posted_at forward.
|
||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25,
|
||
tzinfo=UTC)))
|
||
still_earlier = store.role(posting().uid)
|
||
assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00"
|
||
|
||
|
||
def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None:
|
||
store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
|
||
role = store.role(posting().uid)
|
||
assert role is not None
|
||
assert role.posted_at == role.first_seen
|
||
|
||
|
||
def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> None:
|
||
"""`function` behaves the opposite of every other raw field: a pull may
|
||
only set it before extract has run, and once extract has answered — even
|
||
with a different function than the pull's guess — no later pull may put
|
||
the guess back. The no-flip property across pull, extract, pull."""
|
||
store.upsert_role(posting(function="engineering"))
|
||
uid = posting().uid
|
||
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
|
||
|
||
store.set_extracted(uid, {"function": "sales"})
|
||
assert store.role(uid).function == "sales" # type: ignore[union-attr]
|
||
|
||
# A later pull with the classifier's old (now wrong) guess must not flip
|
||
# it back — the "board wins" COALESCE that every other raw field gets
|
||
# would otherwise put "engineering" straight back on every pull.
|
||
store.upsert_role(posting(function="engineering"))
|
||
assert store.role(uid).function == "sales" # type: ignore[union-attr]
|
||
|
||
|
||
def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None:
|
||
store.upsert_role(posting(function="engineering"))
|
||
uid = posting().uid
|
||
store.set_extracted(uid, {}) # extract answered this role, but not function
|
||
assert store.role(uid).extracted_at is not None # type: ignore[union-attr]
|
||
|
||
# extracted_at alone freezes it, whatever a later pull's guess says.
|
||
store.upsert_role(posting(function="sales"))
|
||
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
|
||
|
||
|
||
def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None:
|
||
store.upsert_role(posting(function="engineering"))
|
||
role = store.role(posting().uid)
|
||
assert role is not None and role.function == "engineering"
|
||
# Before extract, a fresh guess still overwrites the old one, the same
|
||
# "board wins" behaviour as place, country and the rest.
|
||
store.upsert_role(posting(function="sales"))
|
||
role = store.role(posting().uid)
|
||
assert role is not None and role.function == "sales"
|
||
|
||
|
||
def test_company_falls_back_to_the_subscription(store: Store) -> None:
|
||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
|
||
store.upsert_role(posting(), sub)
|
||
role = store.role(posting().uid)
|
||
assert role is not None
|
||
assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id)
|
||
|
||
|
||
def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
|
||
store.upsert_role(posting("1", company="Acme", place="Berlin"))
|
||
clock.tick()
|
||
store.upsert_role(posting("2", company="Acme", place="Munich"))
|
||
clock.tick()
|
||
store.upsert_role(posting("3", company="Other", place="Berlin"))
|
||
assert [r.source_id for r in store.roles()] == ["3", "2", "1"]
|
||
assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"]
|
||
assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"]
|
||
assert [r.source_id for r in store.roles(limit=1)] == ["3"]
|
||
|
||
|
||
def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None:
|
||
"""A role from a trusted board found today but posted months ago must not
|
||
outrank one the app has genuinely known about for longer."""
|
||
store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen
|
||
clock.tick()
|
||
store.upsert_role(posting("old", board="greenhouse",
|
||
published_at=datetime(2020, 1, 1, tzinfo=UTC)))
|
||
assert [r.source_id for r in store.roles()] == ["recent", "old"]
|
||
assert [r.source_id for r in store.roles(days=30)] == ["recent"]
|
||
|
||
|
||
def _spy_sql(monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||
"""Every SQL statement a `Store` runs through `_rows`, in order."""
|
||
captured: list[str] = []
|
||
original = Store._rows
|
||
|
||
def spy(self: Store, sql: str, params: object = ()) -> list[sqlite3.Row]:
|
||
captured.append(sql)
|
||
return original(self, sql, params)
|
||
|
||
monkeypatch.setattr(Store, "_rows", spy)
|
||
return captured
|
||
|
||
|
||
def test_the_newest_feed_query_uses_the_partial_posted_at_index(
|
||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""`Store.roles`' default order (newest, open only) is forced onto
|
||
`idx_roles_posted`: left to the planner's plain, non-STAT4 statistics, it
|
||
can instead pick `idx_roles_closed` and sort every open role, which on
|
||
the live database (15,558 of 15,815 roles open) measured over 30x slower
|
||
than the plan this asserts (see docs/decisions.md)."""
|
||
store.upsert_role(posting("1"))
|
||
sql, _ = store._roles_from(
|
||
company=None, place=None, open_only=True, q=None, function=None, remote=None,
|
||
days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any",
|
||
agent=None, by_score=True, gated=False, judged=None)
|
||
assert "INDEXED BY" not in sql # the hint belongs to roles(), not the shared builder
|
||
|
||
captured = _spy_sql(monkeypatch)
|
||
store.roles(limit=10)
|
||
assert "INDEXED BY idx_roles_posted" in captured[0]
|
||
with sqlite3.connect(store.path) as conn:
|
||
plan = " ".join(row[3] for row in
|
||
conn.execute("EXPLAIN QUERY PLAN " + captured[0], [None, 10, 0]))
|
||
assert "USING INDEX idx_roles_posted" in plan and "TEMP B-TREE" not in plan
|
||
|
||
# open_only=False cannot use it: the index is partial on closed_at IS NULL.
|
||
captured.clear()
|
||
store.roles(limit=10, open_only=False)
|
||
assert "INDEXED BY" not in captured[0]
|
||
|
||
|
||
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
|
||
store.upsert_role(posting("1", company="Acme", place="Berlin",
|
||
function="engineering", remote=True))
|
||
clock.tick(hours=48)
|
||
store.upsert_role(posting("2", company="Acme", place="Munich",
|
||
function="sales"))
|
||
assert [r.source_id for r in store.roles(q="senior engineer")] == ["2", "1"]
|
||
assert [r.source_id for r in store.roles(q="backend nonsense")] == []
|
||
assert [r.source_id for r in store.roles(function="engineering")] == ["1"]
|
||
assert [r.source_id for r in store.roles(remote="remote")] == ["1"]
|
||
assert [r.source_id for r in store.roles(company="ac")] == ["2", "1"]
|
||
assert [r.source_id for r in store.roles(place="berlin")] == ["1"]
|
||
assert [r.source_id for r in store.roles(days=1)] == ["2"]
|
||
assert [r.source_id for r in store.roles(days=30)] == ["2", "1"]
|
||
|
||
|
||
def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> None:
|
||
"""`pay` filters in SQL, the same as the other filters `_roles_from`
|
||
builds, so it narrows what a `limit` is taken from rather than the page
|
||
already built (docs/decisions.md, "filtering before LIMIT")."""
|
||
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
|
||
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
|
||
clock.tick()
|
||
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
|
||
clock.tick()
|
||
store.upsert_role(posting("none"))
|
||
assert {r.source_id for r in store.roles(pay="stated")} == {"normalised", "stated"}
|
||
assert [r.source_id for r in store.roles(pay="normalised")] == ["normalised"]
|
||
assert [r.source_id for r in store.roles(pay="stated_only")] == ["stated"]
|
||
assert {r.source_id for r in store.roles()} == {"normalised", "stated", "none"}
|
||
assert store.count_roles(pay="stated") == 2
|
||
with pytest.raises(ValueError, match="pay is"):
|
||
store.roles(pay="bogus")
|
||
|
||
|
||
def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None:
|
||
agent = store.save_agent("A")
|
||
store.add_company("Acme", key="acme")
|
||
store.set_affinity(agent.id, "acme", -1)
|
||
store.upsert_role(posting("other", company="Other"))
|
||
clock.tick()
|
||
for i in range(3):
|
||
store.upsert_role(posting(f"a{i}", company="Acme"))
|
||
clock.tick()
|
||
# Acme's roles are the three newest, so a plain limit=1 shows only Acme;
|
||
# hide_muted has to reach past all three to find "other" rather than
|
||
# just dropping Acme's rows out of an already-limited page.
|
||
listing = {"agent": agent.id, "by_score": False}
|
||
assert [r.source_id for r in store.roles(limit=1, **listing)] == ["a2"]
|
||
assert [r.source_id for r in store.roles(limit=1, hide_muted=True, **listing)] == ["other"]
|
||
assert {r.source_id for r in store.roles(hide_muted=True, **listing)} == {"other"}
|
||
assert {r.source_id for r in store.roles(**listing)} == {"a0", "a1", "a2", "other"}
|
||
# Hidden is per agent: another agent, or no agent, hides nothing.
|
||
other = store.save_agent("B")
|
||
assert store.count_roles(hide_muted=True, agent=other.id, by_score=False) == 4
|
||
assert store.count_roles(hide_muted=True) == 4
|
||
|
||
|
||
def test_roles_page_by_offset_and_count_the_same_filter(
|
||
store: Store, clock: Clock) -> None:
|
||
for sid in ("1", "2", "3", "4"):
|
||
store.upsert_role(posting(sid, company="Acme" if sid != "4" else "Other"))
|
||
clock.tick()
|
||
assert [r.source_id for r in store.roles(limit=2, offset=1)] == ["3", "2"]
|
||
assert [r.source_id for r in store.roles(limit=2, offset=3)] == ["1"]
|
||
assert store.count_roles() == 4
|
||
assert store.count_roles(company="acme") == 3
|
||
agent = store.save_agent("A")
|
||
store.add_company("Other", key="other")
|
||
store.set_affinity(agent.id, "other", -1)
|
||
assert store.count_roles(hide_muted=True, agent=agent.id, by_score=False) == 3
|
||
|
||
|
||
def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> None:
|
||
for sid in ("up", "down", "fine", "none"):
|
||
store.upsert_role(posting(sid))
|
||
uid = {r.source_id: r.uid for r in store.roles()}
|
||
agent = store.save_agent("A")
|
||
role = store.set_vote(agent.id, uid["up"], 1)
|
||
assert (role.vote, role.voted_at) == (1, clock().isoformat(timespec="seconds"))
|
||
store.set_vote(agent.id, uid["down"], -1)
|
||
store.set_vote(agent.id, uid["fine"], 0)
|
||
assert store.votes(agent.id) == {uid["up"]: 1, uid["down"]: -1, uid["fine"]: 0}
|
||
|
||
def listed(judged: str) -> list[str]:
|
||
return [r.source_id for r in store.roles(agent=agent.id, by_score=False,
|
||
judged=judged)]
|
||
|
||
assert listed("liked") == ["up"]
|
||
assert listed("passed") == ["down"]
|
||
assert listed("unjudged") == ["none"]
|
||
assert store.count_roles(agent=agent.id, by_score=False, judged="unjudged") == 1
|
||
with pytest.raises(ValueError, match="judged"):
|
||
store.roles(judged="maybe")
|
||
|
||
clock.tick()
|
||
changed = store.set_vote(agent.id, uid["up"], -1)
|
||
assert (changed.vote, changed.voted_at) == (-1, clock().isoformat(timespec="seconds"))
|
||
cleared = store.set_vote(agent.id, uid["up"], None)
|
||
assert (cleared.vote, cleared.voted_at) == (None, None)
|
||
assert store.set_vote(agent.id, uid["up"], None).vote is None # clearing twice is fine
|
||
with pytest.raises(ValueError):
|
||
store.set_vote(agent.id, uid["up"], 2)
|
||
with pytest.raises(ValueError):
|
||
store.set_vote(agent.id, uid["up"], True)
|
||
with pytest.raises(KeyError):
|
||
store.set_vote(agent.id, "nope", 1)
|
||
with pytest.raises(KeyError):
|
||
store.set_vote(999, uid["up"], 1)
|
||
|
||
|
||
def test_votes_are_per_agent(store: Store) -> None:
|
||
store.upsert_role(posting("1"))
|
||
uid = posting("1").uid
|
||
one = store.save_agent("One")
|
||
two = store.save_agent("Two")
|
||
store.set_vote(one.id, uid, 1)
|
||
|
||
assert store.role(uid, one.id).vote == 1 # type: ignore[union-attr]
|
||
assert store.role(uid, two.id).vote is None # type: ignore[union-attr]
|
||
assert store.role(uid).vote is None # type: ignore[union-attr]
|
||
assert store.votes(two.id) == {}
|
||
assert [r.vote for r in store.roles(agent=two.id, by_score=False)] == [None]
|
||
assert [r.vote for r in store.roles()] == [None]
|
||
assert [r.source_id for r in store.roles(agent=two.id, by_score=False, judged="liked")] == []
|
||
assert [r.source_id for r in store.roles(judged="liked")] == []
|
||
assert [r.source_id for r in store.roles(judged="unjudged")] == ["1"]
|
||
|
||
store.set_vote(two.id, uid, -1)
|
||
assert store.role(uid, one.id).vote == 1 # type: ignore[union-attr]
|
||
assert store.role(uid, two.id).vote == -1 # type: ignore[union-attr]
|
||
|
||
# Deleting the agent takes its votes with it and leaves the other's.
|
||
store.delete_agent(two.id)
|
||
assert store.votes(one.id) == {uid: 1}
|
||
with store.transaction() as conn:
|
||
assert conn.execute("SELECT COUNT(*) FROM role_votes").fetchone()[0] == 1
|
||
|
||
|
||
def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
|
||
store: Store, clock: Clock) -> None:
|
||
for sid in ("1", "2", "3"):
|
||
store.upsert_role(posting(sid))
|
||
clock.tick()
|
||
uid = {r.source_id: r.uid for r in store.roles()}
|
||
agent = store.save_agent("A")
|
||
store.replace_scores(agent.id, [(uid["1"], 0.9, ""), (uid["2"], 0.1, "")]) # 3 unscored
|
||
store.set_vote(agent.id, uid["1"], 1)
|
||
|
||
newest = store.roles(agent=agent.id, by_score=False)
|
||
assert [r.source_id for r in newest] == ["3", "2", "1"]
|
||
assert [r.score for r in newest] == [None, None, None]
|
||
assert [r.vote for r in newest] == [None, None, 1]
|
||
assert store.count_roles(agent=agent.id, by_score=False) == 3
|
||
assert [r.source_id for r in store.roles(agent=agent.id)] == ["1", "2"]
|
||
|
||
|
||
def test_read_state_is_set_cleared_and_a_vote_marks_read(
|
||
store: Store, clock: Clock) -> None:
|
||
store.upsert_role(posting("1"))
|
||
uid = posting("1").uid
|
||
role = store.role(uid)
|
||
assert role is not None and role.read_at is None
|
||
|
||
read = store.set_read(uid)
|
||
assert read.read_at == clock().isoformat(timespec="seconds")
|
||
clock.tick()
|
||
again = store.set_read(uid)
|
||
assert again.read_at == read.read_at, "reading twice keeps the first stamp"
|
||
|
||
cleared = store.set_read(uid, False)
|
||
assert cleared.read_at is None
|
||
|
||
agent = store.save_agent("A")
|
||
voted = store.set_vote(agent.id, uid, 1)
|
||
assert voted.read_at is not None, "voting is reading"
|
||
stamp = voted.read_at
|
||
unvoted = store.set_vote(agent.id, uid, None)
|
||
assert unvoted.read_at == stamp, "clearing a vote is not un-reading"
|
||
|
||
with pytest.raises(KeyError):
|
||
store.set_read("nope")
|
||
|
||
|
||
def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) -> None:
|
||
store.upsert_role(posting("1"))
|
||
store.upsert_role(posting("2"))
|
||
uid = {r.source_id: r.uid for r in store.roles()}
|
||
store.set_read(uid["1"])
|
||
|
||
assert {r.source_id for r in store.roles()} == {"1", "2"}
|
||
assert {r.source_id for r in store.roles(include_read=False)} == {"2"}
|
||
assert store.count_roles(include_read=False) == 1
|
||
|
||
|
||
def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None:
|
||
store.upsert_role(posting("1"))
|
||
uid = posting("1").uid
|
||
long_tag = "x" * 50
|
||
|
||
tags = store.set_tags(uid, [" Backend Engineer ", "python", "PYTHON", "", " ",
|
||
long_tag, *[f"t{i}" for i in range(15)]])
|
||
|
||
assert tags == sorted(tags)
|
||
assert len(tags) == 12 # capped at _MAX_TAGS
|
||
assert len(set(tags)) == 12 # no repeats
|
||
assert all(len(t) <= 40 for t in tags) # cut to _MAX_TAG_LEN
|
||
assert "backend engineer" in tags and "python" in tags
|
||
role = store.role(uid)
|
||
assert role is not None and role.tags == tags # loaded sorted, with the role
|
||
|
||
with pytest.raises(KeyError):
|
||
store.set_tags("nope", ["x"])
|
||
|
||
|
||
def test_roles_filters_by_tags_any_and_all(store: Store) -> None:
|
||
store.upsert_role(posting("1"))
|
||
store.upsert_role(posting("2"))
|
||
store.upsert_role(posting("3"))
|
||
uid = {r.source_id: r.uid for r in store.roles()}
|
||
store.set_tags(uid["1"], ["python", "backend"])
|
||
store.set_tags(uid["2"], ["python", "frontend"])
|
||
store.set_tags(uid["3"], ["go"])
|
||
|
||
assert {r.source_id for r in store.roles(tags=["python"])} == {"1", "2"}
|
||
assert {r.source_id for r in store.roles(tags=["python", "go"])} == {"1", "2", "3"}
|
||
assert {r.source_id for r in
|
||
store.roles(tags=["python", "backend"], tag_mode="all")} == {"1"}
|
||
assert store.count_roles(tags=["python"]) == 2
|
||
assert store.count_roles(tags=["python", "backend"], tag_mode="all") == 1
|
||
with pytest.raises(ValueError, match="tag_mode"):
|
||
store.roles(tags=["python"], tag_mode="maybe")
|
||
|
||
|
||
def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) -> None:
|
||
store.add_company("Hidden", key="hidden")
|
||
agent = store.save_agent("A")
|
||
store.set_affinity(agent.id, "hidden", -1)
|
||
store.upsert_role(posting("1", company="Acme"))
|
||
store.upsert_role(posting("2", company="Acme"))
|
||
store.upsert_role(posting("3", company="Hidden"))
|
||
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
|
||
store.set_tags(uid["1"], ["python", "backend"])
|
||
store.set_tags(uid["2"], ["python"])
|
||
store.set_tags(uid["3"], ["python", "backend"])
|
||
store.set_read(uid["1"])
|
||
|
||
assert store.tag_counts(agent.id) == [("python", 2, 1), ("backend", 1, 0)]
|
||
assert store.tag_counts(agent.id, show_muted=True) == [("python", 3, 2), ("backend", 2, 1)]
|
||
assert store.tag_counts() == [("python", 3, 2), ("backend", 2, 1)] # no agent, nothing hidden
|
||
|
||
|
||
def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Store) -> None:
|
||
store.upsert_role(posting("a"))
|
||
uid = store.roles()[0].uid
|
||
agent = store.save_agent("A")
|
||
store.delete_agent(agent.id)
|
||
assert store.replace_scores(agent.id, [(uid, 0.9, "")]) == 0
|
||
assert store.scores(agent.id) == {}
|
||
|
||
|
||
def test_a_vote_moves_the_served_score_and_the_order(store: Store) -> None:
|
||
for sid in ("a", "b", "c"):
|
||
store.upsert_role(posting(sid))
|
||
uid = {r.source_id: r.uid for r in store.roles()}
|
||
agent = store.save_agent("A")
|
||
other = store.save_agent("B")
|
||
store.replace_scores(agent.id, [(uid["a"], 0.9, ""), (uid["b"], 0.5, ""),
|
||
(uid["c"], 0.4, "")])
|
||
store.replace_scores(other.id, [(uid["a"], 0.9, ""), (uid["b"], 0.5, ""),
|
||
(uid["c"], 0.4, "")])
|
||
store.set_vote(agent.id, uid["a"], -1) # 0.45
|
||
store.set_vote(agent.id, uid["c"], 1) # 0.48
|
||
ranked = [(r.source_id, round(r.score or 0, 3)) for r in store.roles(agent=agent.id)]
|
||
assert ranked == [("b", 0.5), ("c", 0.48), ("a", 0.45)]
|
||
assert store.scores(agent.id)[uid["a"]][0] == 0.9 # the stored score is kept
|
||
store.set_vote(agent.id, uid["a"], 1)
|
||
assert store.roles(agent=agent.id, limit=1)[0].score == 1.0
|
||
# The other agent's ranking is untouched by these votes.
|
||
assert [(r.source_id, r.score) for r in store.roles(agent=other.id)] == \
|
||
[("a", 0.9), ("b", 0.5), ("c", 0.4)]
|
||
|
||
|
||
def test_mark_closed_closes_what_the_board_no_longer_lists(store: Store) -> None:
|
||
sub = store.add_subscription("ats", "greenhouse:acme")
|
||
other = store.add_subscription("ats", "greenhouse:other")
|
||
for source_id in ("1", "2", "3"):
|
||
store.upsert_role(posting(source_id), sub)
|
||
store.upsert_role(posting("9"), other)
|
||
assert store.mark_closed("ats", ["1", "3"], sub.id) == 1
|
||
assert {r.source_id for r in store.roles()} == {"1", "3", "9"}
|
||
assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"}
|
||
# Listed again: open again.
|
||
store.upsert_role(posting("2"), sub)
|
||
assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"}
|
||
|
||
|
||
def test_an_item_without_a_source_is_refused(store: Store) -> None:
|
||
item = posting()
|
||
item.source = ""
|
||
with pytest.raises(ValueError, match="no source"):
|
||
store.upsert_role(item)
|
||
|
||
|
||
# --------------------------------------------------------------- companies
|
||
|
||
def test_companies_crud(store: Store) -> None:
|
||
acme = store.add_company("Acme GmbH", domain="acme.example")
|
||
assert acme.key == "acme" and acme.affinity == 0 and acme.status == "researching"
|
||
with pytest.raises(ValueError, match="already exists"):
|
||
store.add_company("Acme")
|
||
updated = store.update_company("acme", status="contacted")
|
||
assert updated.status == "contacted"
|
||
with pytest.raises(ValueError, match="not a company field"):
|
||
store.update_company("acme", affinity=2)
|
||
with pytest.raises(ValueError, match="not a company field"):
|
||
store.update_company("acme", created_at="x")
|
||
assert [c.key for c in store.companies()] == ["acme"]
|
||
assert store.remove_company("acme") is True
|
||
assert store.company("acme") is None
|
||
|
||
|
||
def test_a_company_takes_a_reason_from_the_vocabulary_and_a_check_date(store: Store) -> None:
|
||
acme = store.add_company("Acme", hand_reason="robots", checked_at="2026-09-01T00:00:00+00:00")
|
||
assert (acme.hand_reason, acme.checked_at) == ("robots", "2026-09-01T00:00:00+00:00")
|
||
assert store.add_company("Plain").hand_reason == ""
|
||
with pytest.raises(ValueError, match="hand_reason is one of"):
|
||
store.add_company("Odd", hand_reason="lazy")
|
||
assert store.company("odd") is None
|
||
assert store.update_company("acme", hand_reason="").hand_reason == ""
|
||
with pytest.raises(ValueError, match="hand_reason is one of"):
|
||
store.update_company("acme", hand_reason="lazy")
|
||
assert store.company("acme").hand_reason == "" # type: ignore[union-attr]
|
||
|
||
|
||
def test_removing_a_company_takes_every_agents_stance_on_it(store: Store) -> None:
|
||
store.add_company("Acme")
|
||
store.add_company("Other")
|
||
one, two = store.save_agent("One"), store.save_agent("Two")
|
||
store.set_affinity(one.id, "acme", 2)
|
||
store.set_affinity(two.id, "acme", -1)
|
||
store.set_affinity(one.id, "other", 1)
|
||
|
||
assert store.remove_company("acme") is True
|
||
assert store.remove_company("acme") is False
|
||
|
||
# Added again under the same name, it starts neutral for everyone.
|
||
store.add_company("Acme")
|
||
assert store.company("acme", one.id).affinity == 0 # type: ignore[union-attr]
|
||
assert store.company("acme", two.id).affinity == 0 # type: ignore[union-attr]
|
||
assert store.company("other", one.id).affinity == 1 # type: ignore[union-attr]
|
||
|
||
|
||
def test_affinity_is_per_agent(store: Store) -> None:
|
||
store.add_company("Acme", key="acme")
|
||
store.add_company("Other", key="other")
|
||
one = store.save_agent("One")
|
||
two = store.save_agent("Two")
|
||
|
||
assert store.set_affinity(one.id, "acme", 2).affinity == 2
|
||
assert store.company("acme", one.id).affinity == 2 # type: ignore[union-attr]
|
||
assert store.company("acme", two.id).affinity == 0 # type: ignore[union-attr]
|
||
assert store.company("acme").affinity == 0 # type: ignore[union-attr]
|
||
assert [(c.key, c.affinity) for c in store.companies(one.id)] == [("acme", 2), ("other", 0)]
|
||
assert [c.affinity for c in store.companies(two.id)] == [0, 0]
|
||
assert [c.affinity for c in store.companies()] == [0, 0]
|
||
|
||
store.set_affinity(two.id, "acme", -1)
|
||
assert store.company("acme", one.id).affinity == 2 # type: ignore[union-attr]
|
||
assert store.company("acme", two.id).affinity == -1 # type: ignore[union-attr]
|
||
|
||
# 0 deletes the row rather than keeping a neutral one.
|
||
assert store.set_affinity(one.id, "acme", 0).affinity == 0
|
||
with store.transaction() as conn:
|
||
rows = conn.execute("SELECT agent_id, company_key FROM agent_companies").fetchall()
|
||
assert [tuple(r) for r in rows] == [(two.id, "acme")]
|
||
|
||
with pytest.raises(ValueError, match="affinity"):
|
||
store.set_affinity(one.id, "acme", 5)
|
||
with pytest.raises(ValueError, match="affinity"):
|
||
store.set_affinity(one.id, "acme", True) # type: ignore[arg-type]
|
||
with pytest.raises(KeyError):
|
||
store.set_affinity(one.id, "nope", 1)
|
||
with pytest.raises(KeyError):
|
||
store.set_affinity(999, "acme", 1)
|
||
|
||
store.delete_agent(two.id)
|
||
with store.transaction() as conn:
|
||
assert conn.execute("SELECT COUNT(*) FROM agent_companies").fetchone()[0] == 0
|
||
|
||
|
||
def test_the_default_agent_is_the_lowest_enabled_id(store: Store) -> None:
|
||
assert store.default_agent() is None
|
||
off = store.save_agent("Off", enabled=False)
|
||
assert store.default_agent() == off
|
||
on = store.save_agent("On")
|
||
assert store.default_agent() == on
|
||
later = store.save_agent("Later")
|
||
assert store.default_agent() == on
|
||
store.delete_agent(on.id)
|
||
assert store.default_agent() == later
|
||
|
||
|
||
def test_open_role_counts_and_company_boards(store: Store) -> None:
|
||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
||
store.add_company("Acme", key="acme")
|
||
store.upsert_role(posting("1", company="Acme", place="Berlin"), sub)
|
||
store.upsert_role(posting("2", company="Acme", place="Munich"), sub)
|
||
store.mark_closed("ats", ["1"], sub.id)
|
||
assert store.open_role_counts() == {"acme": 1}
|
||
assert store.company_boards() == {"acme": "greenhouse"}
|
||
|
||
|
||
def test_set_company_board_creates_the_row_and_its_board_together(store: Store) -> None:
|
||
company, sub, closed = store.set_company_board(
|
||
"acme", "greenhouse:acme", host="greenhouse", name="Acme",
|
||
fields={"ats": "greenhouse", "ats_slug": "acme", "country": "DK"})
|
||
assert (company.name, company.ats, company.ats_slug, company.country) == (
|
||
"Acme", "greenhouse", "acme", "DK")
|
||
assert sub is not None and closed == 0
|
||
assert (sub.target, sub.label, sub.company, sub.board, sub.show) == (
|
||
"greenhouse:acme", "Acme", "Acme", "greenhouse", 1)
|
||
assert store.company_subscription("acme") == sub
|
||
assert store.company_boards() == {"acme": "greenhouse"}
|
||
|
||
|
||
def test_a_board_clash_leaves_no_company_row(store: Store) -> None:
|
||
"""The subscription insert fails on UNIQUE (source, target), and the
|
||
company row written a statement earlier goes with it."""
|
||
store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme")
|
||
with pytest.raises(ValueError, match="already has 'greenhouse:acme'"):
|
||
store.set_company_board("beta", "greenhouse:acme", host="greenhouse", name="Beta")
|
||
assert store.company("beta") is None
|
||
assert [s.company for s in store.subscriptions(source="ats")] == ["Acme"]
|
||
|
||
|
||
def test_a_clash_on_replace_leaves_the_old_board_and_its_roles(store: Store) -> None:
|
||
store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme")
|
||
_, beta, _ = store.set_company_board("beta", "lever:beta", host="lever", name="Beta")
|
||
assert beta is not None
|
||
store.upsert_role(posting("1", company="Beta"), beta)
|
||
with pytest.raises(ValueError):
|
||
store.set_company_board("beta", "greenhouse:acme", host="greenhouse",
|
||
fields={"ats": "greenhouse"})
|
||
assert store.company_subscription("beta") == beta
|
||
assert store.company("beta").ats == "" # type: ignore[union-attr]
|
||
assert [r.source_id for r in store.roles()] == ["1"] # still open
|
||
|
||
|
||
def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles(
|
||
store: Store) -> None:
|
||
_, old, _ = store.set_company_board("acme", "greenhouse:acme", host="greenhouse",
|
||
name="Acme")
|
||
assert old is not None
|
||
store.update_subscription(old.id, fetch=False, target_id="gh-1")
|
||
store.upsert_role(posting("1", company="Acme"), old)
|
||
store.upsert_role(posting("2", company="Acme"), old)
|
||
|
||
company, new, closed = store.set_company_board("acme", "ashby:acme", host="ashby",
|
||
fields={"ats": "ashby"})
|
||
assert closed == 2 and store.roles() == []
|
||
assert new is not None and new.id == old.id
|
||
assert (new.target, new.board, new.target_id, new.fetch, new.subscribed) == (
|
||
"ashby:acme", "ashby", "", 1, 1)
|
||
assert company.ats == "ashby"
|
||
# The same board again closes nothing.
|
||
store.upsert_role(posting("3", company="Acme"), new)
|
||
assert store.set_company_board("acme", "ashby:acme", host="ashby")[2] == 0
|
||
assert [r.source_id for r in store.roles()] == ["3"]
|
||
|
||
|
||
def test_detaching_a_board_hides_it_closes_its_roles_and_keeps_the_row(
|
||
store: Store) -> None:
|
||
_, sub, _ = store.set_company_board("acme", "greenhouse:acme", host="greenhouse",
|
||
name="Acme")
|
||
assert sub is not None
|
||
store.upsert_role(posting("1", company="Acme"), sub)
|
||
company, detached, closed = store.set_company_board("acme", "")
|
||
assert closed == 1 and company.key == "acme"
|
||
assert detached is not None
|
||
assert (detached.subscribed, detached.fetch, detached.show) == (0, 0, 0)
|
||
assert store.company_boards() == {}
|
||
# The next board reuses the row rather than clashing on its label.
|
||
_, again, _ = store.set_company_board("acme", "lever:acme", host="lever")
|
||
assert again is not None and again.id == sub.id and again.show == 1
|
||
with pytest.raises(KeyError):
|
||
store.set_company_board("nope", "")
|
||
|
||
|
||
# ----------------------------------------------------------- subscriptions
|
||
|
||
def test_subscriptions_crud(store: Store) -> None:
|
||
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
||
company="Acme", board="Greenhouse", params={"q": 1})
|
||
assert (sub.company, sub.board, sub.params, sub.fetch) == \
|
||
("Acme", "Greenhouse", {"q": 1}, 1)
|
||
with pytest.raises(ValueError):
|
||
store.add_subscription("ats", "greenhouse:acme")
|
||
assert store.subscription_for("ats", "Acme") == sub
|
||
store.update_subscription(sub.id, fetch=False)
|
||
assert store.subscriptions(fetchable=True) == []
|
||
assert [s.id for s in store.subscriptions(source="ats")] == [sub.id]
|
||
gone = store.unsubscribe(sub.id)
|
||
assert (gone.subscribed, gone.fetch, gone.show) == (0, 0, 0)
|
||
|
||
|
||
# ------------------------------------------------------------ applications
|
||
|
||
def test_a_status_change_writes_an_event(store: Store, clock: Clock) -> None:
|
||
saved = store.save_application(Application(company_key="acme", role_title="Engineer"))
|
||
assert saved.id and saved.status == "saved"
|
||
clock.tick()
|
||
store.set_application_status(saved.id, "applied")
|
||
clock.tick()
|
||
store.save_application(saved.model_copy(update={"status": "applied", "notes": "x"}))
|
||
clock.tick()
|
||
store.set_application_status(saved.id, "screen")
|
||
events = store.application_events(saved.id)
|
||
assert [e.status for e in events] == ["saved", "applied", "screen"]
|
||
assert events[0].at < events[1].at < events[2].at
|
||
assert [a.notes for a in store.applications()] == ["x"]
|
||
|
||
|
||
def test_an_invalid_status_is_refused(store: Store) -> None:
|
||
saved = store.save_application(Application(role_title="Engineer"))
|
||
with pytest.raises(ValueError, match="not an application status"):
|
||
store.set_application_status(saved.id, "ghosted")
|
||
with pytest.raises(ValidationError):
|
||
Application(status="ghosted")
|
||
with pytest.raises(sqlite3.IntegrityError), store.transaction() as conn:
|
||
conn.execute("UPDATE applications SET status='ghosted' WHERE id=?", (saved.id,))
|
||
assert [e.status for e in store.application_events(saved.id)] == ["saved"]
|
||
|
||
|
||
def test_events_are_append_only(store: Store) -> None:
|
||
saved = store.save_application(Application(role_title="Engineer"))
|
||
with pytest.raises(sqlite3.IntegrityError, match="append-only"), \
|
||
store.transaction() as conn:
|
||
conn.execute("UPDATE application_events SET status='offer'")
|
||
assert store.application_events(saved.id)[0].status == "saved"
|
||
|
||
|
||
def test_an_application_may_point_at_a_role(store: Store) -> None:
|
||
store.upsert_role(posting())
|
||
saved = store.save_application(Application(role_uid=posting().uid, status="applied"))
|
||
assert saved.role_uid == posting().uid
|
||
assert [e.status for e in store.application_events(saved.id)] == ["applied"]
|
||
|
||
|
||
# -------------------------------------------------- summary and posting text
|
||
|
||
def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
|
||
store.upsert_role(posting())
|
||
role = store.set_article_text(posting().uid, "The whole posting.")
|
||
assert role is not None
|
||
assert role.article_text == "The whole posting." and role.article_at == store.now()
|
||
assert store.set_article_text("nope", "x") is None
|
||
|
||
|
||
def test_a_summary_is_written_once(store: Store) -> None:
|
||
store.upsert_role(posting())
|
||
uid = posting().uid
|
||
assert store.set_summary(uid, "First.") is True
|
||
assert store.set_summary(uid, "Second.") is False
|
||
role = store.role(uid)
|
||
assert role is not None and role.summary == "First." and role.summarised_at
|
||
assert store.set_summary("nope", "x") is False
|
||
|
||
|
||
def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer(
|
||
store: Store) -> None:
|
||
store.upsert_role(posting())
|
||
uid = posting().uid
|
||
assert store.set_summary(uid, "First.", ["Python", "backend"]) is True
|
||
role = store.role(uid)
|
||
assert role is not None and role.tags == ["backend", "python"]
|
||
|
||
# A second write is rejected (already summarised), and does not touch tags.
|
||
assert store.set_summary(uid, "Second.", ["go"]) is False
|
||
role = store.role(uid)
|
||
assert role is not None and role.tags == ["backend", "python"]
|
||
|
||
|
||
def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
|
||
store: Store) -> None:
|
||
"""Each agent contributes its own top, as deep and as low as its own
|
||
`limits` say; a closed role and a gated score are in nobody's top. A
|
||
hidden company needs no case: `score` writes no rows for it under the
|
||
agent that hid it. An agent saved before v8 has `limits` of '{}', which
|
||
reads as the defaults."""
|
||
for sid in "abcd":
|
||
store.upsert_role(posting(sid))
|
||
a, b, c, d = (posting(s).uid for s in "abcd")
|
||
with store.transaction() as conn:
|
||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
|
||
one = store.save_agent("One", "python backend",
|
||
limits=Limits(summarise_top=1, summarise_min=0))
|
||
two = store.save_agent("Two", "python backend",
|
||
limits=Limits(summarise_top=1, summarise_min=0))
|
||
store.replace_scores(one.id, [(d, 0.99, ""), (a, 0.9, ""), (b, 0.5, "")])
|
||
store.replace_scores(two.id, [(a, 0.95, "remote"), (c, 0.7, ""), (b, 0.8, "")])
|
||
|
||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||
|
||
# Two goes deeper: its second, c, joins; One's second, b, was in already.
|
||
store.save_agent("Two", "python backend", agent_id=two.id,
|
||
limits=Limits(summarise_top=2, summarise_min=0))
|
||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||
assert store.summarise_pending() == 3
|
||
assert [r.uid for r in store.summarise_waiting(10, uids=[d])] == [d]
|
||
# The floor is Two's own: c at 0.7 drops, One's b at 0.5 stays.
|
||
store.save_agent("Two", "python backend", agent_id=two.id,
|
||
limits=Limits(summarise_top=2, summarise_min=0.75))
|
||
store.save_agent("One", "python backend", agent_id=one.id,
|
||
limits=Limits(summarise_top=2, summarise_min=0))
|
||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||
# '{}' is the defaults, 30 at 0.65: One's b at 0.5 drops, but Two's b at
|
||
# 0.8 and c at 0.7 clear it.
|
||
with store.transaction() as conn:
|
||
conn.execute("UPDATE agents SET limits = '{}'")
|
||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||
assert store.agents()[0].limits == Limits()
|