Store copies of one posting as roles.dup_of and leave them out of the feed, the counts and the model stages
Migration 18 adds roles.dup_of with a partial index and runs the first rebuild. Store.rebuild_dupes feeds app.dupes.assign the open roles of every (company_key, title_key) with more than one, rewrites dup_of where it changed, clears it on closed roles, and hands read_at and votes to a changed group's kept role. The pull's settle and `app dupes` run it. RoleRow.copies counts a row's open copies; Role.copies lists them and Role.dup_of names a copy's kept role. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
55e353e764
commit
545c932f7b
10 files changed
+490
-41
No files matched your search
@@ -92,6 +92,13 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- whose description is only a teaser.
|
||||
article_text TEXT NOT NULL DEFAULT '',
|
||||
article_at TEXT,
|
||||
-- The uid of the kept role when this open role is a copy of one posting
|
||||
-- (app/dupes.py), NULL on the kept role, on a role without copies and on
|
||||
-- every closed role. Written only by Store.rebuild_dupes, at the end of
|
||||
-- every pull; the feed, the counts and the model stages skip a copy.
|
||||
-- Not the last column: SQLite cannot drop a last column that a comment
|
||||
-- precedes, which the migration tests do.
|
||||
dup_of TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
salary_text TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
@@ -292,6 +299,10 @@ CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote);
|
||||
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
|
||||
-- The copies of a kept role, for `Store.role` and the feed's copy counts.
|
||||
-- Partial: all but about 3% of roles are no copy.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
ON roles(dup_of) WHERE dup_of IS NOT NULL;
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -455,6 +466,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
# A Python step: the fold and the title classifier are Python, as in 11.
|
||||
(17, "roles.place_fold for the whole-word place filter, and roles.seniority from "
|
||||
"the title where extract has not read the role", lambda conn: _fold_places(conn)),
|
||||
# A Python step: the first rebuild runs the rule in app/dupes.py.
|
||||
(18, "roles.dup_of: one posting listed several times is one role, the others "
|
||||
"copies of it", lambda conn: _add_dup_of(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -566,6 +580,18 @@ def _fold_places(conn: sqlite3.Connection) -> None:
|
||||
len(folds), len(place_updates), len(level_updates))
|
||||
|
||||
|
||||
def _add_dup_of(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 18's step. Adds `roles.dup_of` and runs the first rebuild
|
||||
(`store.rebuild_dupes`) inside `_apply`'s transaction, so a migrated file
|
||||
hides its copies from the first open rather than from the next pull.
|
||||
Imported here, not at the top: `store` imports this module."""
|
||||
from app.storage.store import rebuild_dupes
|
||||
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN dup_of TEXT")
|
||||
counts = rebuild_dupes(conn)
|
||||
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
Reference in new issue
Block a user