Store copies of one posting as roles.dup_of and leave them out of the feed, the counts and the model stages

Migration 18 adds roles.dup_of with a partial index and runs the first rebuild. Store.rebuild_dupes feeds app.dupes.assign the open roles of every (company_key, title_key) with more than one, rewrites dup_of where it changed, clears it on closed roles, and hands read_at and votes to a changed group's kept role. The pull's settle and `app dupes` run it. RoleRow.copies counts a row's open copies; Role.copies lists them and Role.dup_of names a copy's kept role.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 23:09:20 +02:00
1 parent 55e353e764
commit 545c932f7b
10 files changed
+490 -41

No files matched your search

+177
View File
@@ -0,0 +1,177 @@
"""`roles.dup_of`: migration 18, `Store.rebuild_dupes`, and what a copy is left out of."""
from __future__ import annotations
import sqlite3
from datetime import UTC, datetime
from pathlib import Path
import numpy as np
import pytest
from desk.models import Item
from app.storage import Store
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def role(source_id: str, *, company: str = "SAP", title: str = "Forward Deployed Engineer",
place: str = "Garching", body: str = "The posting.", day: int = 20,
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://jobs.example/{source_id}",
title=title, body=body, published_at=datetime(2026, 9, day, tzinfo=UTC),
raw={"company": company, "place": place, "board": "successfactors", **raw})
item.source = "ats"
return item
def dup_of(store: Store) -> dict[str, str | None]:
with sqlite3.connect(store.path) as conn:
return dict(conn.execute("SELECT uid, dup_of FROM roles"))
def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart(
store: Store) -> None:
"""SAP's repeats: three Garching rows are one kept role (the one with
the description) and two copies; New York is its own role."""
garching = [role("1", body=""), role("2"), role("3", body="")]
york = role("4", place="New York, NY, US")
for item in (*garching, york):
store.upsert_role(item)
assert store.rebuild_dupes() == (1, 2, 2)
kept = garching[1].uid
assert dup_of(store) == {garching[0].uid: kept, kept: None,
garching[2].uid: kept, york.uid: None}
assert store.rebuild_dupes().changed == 0 # stable between runs
def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store) -> None:
board = role("board", company="Trackunit", title="Backend Engineer", place="Aalborg",
board="teamtailor", body="")
search = role("search", company="Trackunit", title="Backend Engineer",
place="Aalborg, Denmark", board=None, provider="jobindex")
for item in (search, board):
store.upsert_role(item)
store.rebuild_dupes()
assert dup_of(store) == {board.uid: None, search.uid: board.uid}
def test_berlin_and_munich_stay_two_roles(store: Store) -> None:
for item in (role("b", place="Berlin"), role("m", place="Munich")):
store.upsert_role(item)
assert store.rebuild_dupes() == (0, 0, 0)
def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared(
store: Store) -> None:
first, second, third = role("1", day=18), role("2", day=19), role("3", day=20)
for item in (first, second, third):
store.upsert_role(item)
store.rebuild_dupes()
assert dup_of(store)[second.uid] == first.uid
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,))
store.rebuild_dupes()
assert dup_of(store) == {first.uid: None, second.uid: None, third.uid: second.uid}
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-26' WHERE uid = ?", (third.uid,))
store.rebuild_dupes()
assert set(dup_of(store).values()) == {None}
def test_read_and_votes_move_to_the_kept_role(store: Store) -> None:
"""A copy read and liked before it was known as one: the kept role
becomes read and liked. An agent that voted on the kept role keeps its
own vote, and a later unread on the kept role is not undone."""
one, two = store.save_agent("One"), store.save_agent("Two")
kept, copy = role("kept", day=18), role("copy", day=19)
store.upsert_role(copy)
store.set_vote(one.id, copy.uid, 1)
store.set_vote(two.id, copy.uid, -1)
store.upsert_role(kept)
store.set_vote(two.id, kept.uid, 0)
store.rebuild_dupes()
read = store.role(copy.uid).read_at # type: ignore[union-attr]
assert store.role(kept.uid).read_at == read # type: ignore[union-attr]
assert store.votes(one.id) == {kept.uid: 1}
assert store.votes(two.id) == {kept.uid: 0, copy.uid: -1}
store.set_read(kept.uid, False)
store.rebuild_dupes()
assert store.role(kept.uid).read_at is None # type: ignore[union-attr]
def test_a_closed_kept_role_hands_its_read_and_vote_to_the_next(store: Store) -> None:
agent = store.save_agent("One")
first, second = role("1", day=18), role("2", day=19)
for item in (first, second):
store.upsert_role(item)
store.rebuild_dupes()
store.set_vote(agent.id, first.uid, 1)
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,))
# A third copy keeps the group alive once the first has closed.
store.upsert_role(role("3", day=20))
store.rebuild_dupes()
assert store.votes(agent.id) == {second.uid: 1}
assert store.role(second.uid).read_at is not None # type: ignore[union-attr]
def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None:
kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin")
for item in (kept, copy, other):
store.upsert_role(item)
store.set_tags(item.uid, ["python"])
store.rebuild_dupes()
agent = store.save_agent("A", intent="engineering")
assert {r.uid for r in store.roles()} == {kept.uid, other.uid}
rows = {r.uid: r.copies for r in store.roles_page()}
assert rows == {kept.uid: 1, other.uid: 0}
assert store.count_roles() == 2
assert sum(n for _, n in store.role_facets()["seniority"]) == 2
assert store.tag_counts() == [("python", 2, 2)]
assert {c.uid for c in store.score_candidates(agent.id, [])} == {kept.uid, other.uid}
assert {r.uid for r in store.roles_to_embed("m", 10)} == {kept.uid, other.uid}
assert store.open_role_counts() == {"sap": 2}
assert [count for _, _, count, _ in store.open_companies()] == [2]
store.replace_scores(agent.id, [(u, 0.9, "") for u in (kept.uid, copy.uid, other.uid)])
assert {r.uid for r in store.extract_waiting(10)} == {kept.uid, other.uid}
assert {r.uid for r in store.summarise_waiting(10)} == {kept.uid, other.uid}
# The copy still opens by uid, and the kept role lists it.
assert store.role(copy.uid).dup_of == kept.uid # type: ignore[union-attr]
assert [c.uid for c in store.role(kept.uid).copies] == [copy.uid] # type: ignore[union-attr]
assert store.role(other.uid).copies == [] # type: ignore[union-attr]
def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
path = tmp_path / "v17.db"
store = Store(path)
first, second = role("1", day=18), role("2", day=19)
for item in (first, second):
store.upsert_role(item)
store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))])
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
conn.execute("PRAGMA user_version = 17")
migrated = Store(path)
assert migrated.migrations_applied == [18]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
).fetchone() is not None
+34 -5
View File
@@ -31,11 +31,11 @@ class FakeConfig:
searches: ClassVar[list[object]] = []
def _item(board: str, slug: str, job_id: str) -> Item:
def _item(board: str, slug: str, job_id: str, company: str = "Acme") -> Item:
item = Item(
source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}",
title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "place": "Berlin", "department": "",
raw={"company": company, "place": "Berlin", "department": "",
"function": "engineering", "remote": None, "salary_text": "",
"board": board, "slug": slug})
item.source = "ats"
@@ -126,7 +126,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
store.upsert_role(_item("greenhouse", "acme", "2"), sub)
store.upsert_role(_item("greenhouse", "other", "9"), other)
store.upsert_role(_item("greenhouse", "other", "9", company="Other"), other)
# acme answered this run and now lists only "1"; other's board failed
# (or its batch never got to it) and stamped nothing.
@@ -140,7 +140,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 1, "closed": 1}
"roles": 1, "closed": 1, "copies": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -161,10 +161,39 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 0, "closed": 0}
"roles": 0, "closed": 0, "copies": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
"""The settle's last step: a board's repeats of one job are one role,
and when the kept one closes the next copy is kept in its place."""
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
company="Acme", board="greenhouse")
for job_id in ("1", "2", "3"):
store.upsert_role(_item("greenhouse", "acme", job_id), sub)
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:complete:greenhouse:acme"] = "1"
store.state["ats:ids:greenhouse:acme"] = json.dumps(
["greenhouse:acme:1", "greenhouse:acme:2", "greenhouse:acme:3"])
class FakeTally:
results = ({"roles": 3, "run": "run-1"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result["copies"] == 2
[kept] = store.roles_page()
assert kept.copies == 2
gone = store.role(kept.uid).source_id # type: ignore[union-attr]
store.state["ats:ids:greenhouse:acme"] = json.dumps(
[f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone])
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert (result["closed"], result["copies"]) == (1, 1)
[after] = store.roles_page()
assert after.uid != kept.uid and after.copies == 1
# ---------------------------------------------- whole pulls, fetch stubbed
def _row(job_id: str) -> dict[str, object]:
+12 -5
View File
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 17
assert LATEST == 18
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -88,8 +88,15 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
def _undo_v18(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v17 shape: no copies."""
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
def _undo_v17(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v16 shape: no folded place on a role."""
_undo_v18(conn)
conn.execute("ALTER TABLE roles DROP COLUMN place_fold")
@@ -159,7 +166,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17]
assert migrated.migrations_applied == [15, 16, 17, 18]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -364,7 +371,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17]
assert migrated.migrations_applied == [14, 15, 16, 17, 18]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -388,7 +395,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17]
assert migrated.migrations_applied == [16, 17, 18]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -417,7 +424,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17]
assert migrated.migrations_applied == [17, 18]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
+27 -1
View File
@@ -1159,7 +1159,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
"seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state",
"original_url", "posted_at", "score", "gated_reason", "prompt", "vote",
"voted_at", "read_at", "tags", "excerpt",
"voted_at", "read_at", "tags", "excerpt", "copies",
}
assert row["uid"] == uid
assert row["company"] == "Acme" and row["place"] == "Berlin, Germany"
@@ -1644,3 +1644,29 @@ def test_summarise_queues_one_role_and_the_job_writes_it(
assert asked == [[uid]]
assert store.role(uid).summary == "Backend role in Berlin, Python." # type: ignore[union-attr]
assert client.post(f"/api/roles/{uid}/summarise").status_code == 409
def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
client: TestClient, store: Store) -> None:
"""The contract the reader and the feed row build on: `copies` on a row
is a count, on `GET /api/roles/{uid}` the list of open copies, and a copy
still opens, with `dup_of`."""
board = posting("board", company="Acme", place="Ballerup", board="greenhouse")
board.body = "The whole posting."
search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex")
for item in (board, search, posting("alone", company="Other")):
store.upsert_role(item)
store.rebuild_dupes()
page = client.get("/api/roles").json()
assert page["total"] == 2
assert {r["uid"]: r["copies"] for r in page["items"]} == {
board.uid: 1, posting("alone").uid: 0}
kept = client.get(f"/api/roles/{board.uid}").json()
assert kept["dup_of"] is None
assert kept["copies"] == [{
"uid": search.uid, "host": "jobindex", "place": "Ballerup, Denmark",
"original_url": search.location, "posted_at": kept["copies"][0]["posted_at"]}]
copy = client.get(f"/api/roles/{search.uid}").json()
assert copy["dup_of"] == board.uid and copy["copies"] == []