Merge branch 'wp-i-dupes-storage'

This commit is contained in:
Wolf committed 2026-09-29 23:09:52 +02:00
commit fafd3c7636
10 files changed
+490 -41

No files matched your search

+17
View File
@@ -7,6 +7,7 @@
app searches add|list|remove
app agents list|prompt
app stages [extract|summarise on|off]
app dupes
app db init
`app job` queues the work on a queue of its own and drains it here, so a
@@ -155,6 +156,8 @@ def build_parser() -> argparse.ArgumentParser:
stages.add_argument("name", nargs="?", choices=["extract", "summarise"])
stages.add_argument("state", nargs="?", choices=["on", "off"])
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
return parser
@@ -179,6 +182,8 @@ def main(argv: list[str] | None = None) -> int:
return run_agents(args)
if args.command == "stages":
return run_stages(args)
if args.command == "dupes":
return run_dupes(args)
return init_db(args)
except (FileNotFoundError, config_mod.ConfigError) as exc:
print(exc, file=sys.stderr)
@@ -626,6 +631,18 @@ def run_stages(args: argparse.Namespace) -> int:
return 0
def run_dupes(args: argparse.Namespace) -> int:
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
now. Prints the groups (kept roles with a copy), the copies the feed
hides and the rows whose `dup_of` changed, as JSON."""
from app.storage import Store
config = config_mod.load(args.config)
counts = Store(config.database).rebuild_dupes()
print(json.dumps(counts._asdict()))
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from app.storage import LATEST, Store
+7 -1
View File
@@ -219,6 +219,11 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
After the closes, `Store.rebuild_dupes` recomputes which open roles are
copies of one posting, so a closed kept role hands its group on and the
roles this run added join theirs; `copies` is how many open roles it
leaves hidden as copies.
"""
answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
@@ -252,9 +257,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
closed += store.close_unseen("search", stale)
if run_marker:
release_run(store, run_marker)
copies = store.rebuild_dupes().copies
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed,
"incomplete": incomplete, "roles": roles, "closed": closed}
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
+23 -1
View File
@@ -763,6 +763,18 @@ JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
# ------------------------------------------------------------------ models
class RoleCopy(Model):
"""One open copy of a kept role, as the reader lists it under "Also
posted": where it is listed (`host`, the search provider or the board's
ATS) and the link that opens it there."""
uid: str
host: str = ""
place: str = ""
original_url: str = ""
posted_at: str = ""
class Role(Model):
"""A row of `roles`: one posting on one board."""
@@ -819,6 +831,13 @@ class Role(Model):
#: The open vocabulary `summarise` read off the posting, sorted. Not a
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
tags: list[str] = []
#: The uid of the kept role when this one is a copy of the same posting
#: (`app.dupes`, `Store.rebuild_dupes`), else None. A copy is left out of
#: the feed and the counts but still opens by uid.
dup_of: str | None = None
#: The open copies of this role, oldest first, when it is the kept one;
#: empty otherwise. Not a column: filled by `Store.role`.
copies: list[RoleCopy] = []
@computed_field # type: ignore[prop-decorator]
@property
@@ -895,7 +914,8 @@ class RoleRow(Model):
derived as on `Role`. `original_url` is one short string, kept so the
`o` key opens a selected row's posting without opening the reader.
`seniority` is the title's level until extract reads the posting, ''
when neither says."""
when neither says. `copies` counts the open copies the feed hides behind
this row."""
uid: str
company_key: str = ""
@@ -920,6 +940,8 @@ class RoleRow(Model):
read_at: str | None = None
tags: list[str] = []
excerpt: str = ""
#: How many open roles are copies of this one (`Role.copies`), 0 when none.
copies: int = 0
@computed_field # type: ignore[prop-decorator]
@property
+26
View File
@@ -92,6 +92,13 @@ CREATE TABLE IF NOT EXISTS roles (
-- whose description is only a teaser.
article_text TEXT NOT NULL DEFAULT '',
article_at TEXT,
-- The uid of the kept role when this open role is a copy of one posting
-- (app/dupes.py), NULL on the kept role, on a role without copies and on
-- every closed role. Written only by Store.rebuild_dupes, at the end of
-- every pull; the feed, the counts and the model stages skip a copy.
-- Not the last column: SQLite cannot drop a last column that a comment
-- precedes, which the migration tests do.
dup_of TEXT,
created_at TEXT NOT NULL,
salary_text TEXT NOT NULL DEFAULT ''
);
@@ -292,6 +299,10 @@ CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote);
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
-- The copies of a kept role, for `Store.role` and the feed's copy counts.
-- Partial: all but about 3% of roles are no copy.
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
ON roles(dup_of) WHERE dup_of IS NOT NULL;
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -455,6 +466,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
# A Python step: the fold and the title classifier are Python, as in 11.
(17, "roles.place_fold for the whole-word place filter, and roles.seniority from "
"the title where extract has not read the role", lambda conn: _fold_places(conn)),
# A Python step: the first rebuild runs the rule in app/dupes.py.
(18, "roles.dup_of: one posting listed several times is one role, the others "
"copies of it", lambda conn: _add_dup_of(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -566,6 +580,18 @@ def _fold_places(conn: sqlite3.Connection) -> None:
len(folds), len(place_updates), len(level_updates))
def _add_dup_of(conn: sqlite3.Connection) -> None:
"""Migration 18's step. Adds `roles.dup_of` and runs the first rebuild
(`store.rebuild_dupes`) inside `_apply`'s transaction, so a migrated file
hides its copies from the first open rather than from the next pull.
Imported here, not at the top: `store` imports this module."""
from app.storage.store import rebuild_dupes
conn.execute("ALTER TABLE roles ADD COLUMN dup_of TEXT")
counts = rebuild_dupes(conn)
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
+160 -26
View File
@@ -14,16 +14,17 @@ from collections.abc import Callable, Iterator, MutableMapping, Sequence
from contextlib import closing, contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, Literal
from typing import Any, Literal, NamedTuple
from uuid import uuid4
import numpy as np
from desk.models import Item
from app import places
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
ScoredRole,
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
)
@@ -50,8 +51,9 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
#: never answers is otherwise paid for on every pull for as long as it exists.
GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: never answered, still open, not given up.
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
#: A role `extract` has yet to read: never answered, still open, no copy of
#: another (`rebuild_dupes`), not given up.
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL AND dup_of IS NULL "
f"AND extract_tries < {GIVE_UP_AFTER}")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
@@ -226,6 +228,93 @@ def _int_or_none(value: Any) -> int | None:
return None
#: The host `app.dupes.Copy.host` and `RoleCopy.host` read: the provider of
#: a search's row, else the ATS a board runs on, as `posted_host` reads it.
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
class DupeCounts(NamedTuple):
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
open roles that are a copy, and rows whose `dup_of` it rewrote."""
groups: int
copies: int
changed: int
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
"""Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside
the caller's transaction: `Store.rebuild_dupes` and migration 18.
Only the open roles of a `(company_key, title_key)` with more than one
open role can be a copy, a few thousand of the ~34k, so only those are
read. A row whose `dup_of` differs from the rule's answer is rewritten,
and a `dup_of` left on a closed role, or on one no longer in such a pair,
is cleared: a copy never outlives its posting, and a closed kept role
hands its group to the next copy here.
Where a group changed (a member's `dup_of` was rewritten), its kept role
takes over what was done to the posting under another uid. The sources
are the group's members and the roles they were copies of before, so a
closed kept role still hands over. The kept role takes the earliest
`read_at` of them when it has none, and each agent's latest vote among
them moves to it unless that agent already voted on it; a vote that
cannot move stays where it is. An unchanged group is not touched, so
marking a kept role unread again sticks. Applications keep their
`role_uid`, and scores of a copy go at the agent's next `score`."""
rows = conn.execute(
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
description != '' AS described, posted_at
FROM roles
WHERE closed_at IS NULL AND company_key != ''
AND (company_key, title_key) IN (
SELECT company_key, title_key FROM roles
WHERE closed_at IS NULL AND company_key != ''
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
new = dupes.assign(
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None)
for r in rows)
old: dict[str, str | None] = dict.fromkeys(new)
old.update((r[0], r[1]) for r in conn.execute(
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
changed = {uid for _, uid in updates}
members: dict[str, set[str]] = {}
for uid, dup in new.items():
members.setdefault(dup or uid, set()).add(uid)
groups = {kept: group for kept, group in members.items() if len(group) > 1}
for kept, group in groups.items():
if not group & changed:
continue
sources = (group | {old.get(uid) or uid for uid in group}) - {kept}
_carry(conn, kept, sources)
return DupeCounts(groups=len(groups), copies=sum(dup is not None for dup in new.values()),
changed=len(updates))
def _carry(conn: sqlite3.Connection, kept: str, sources: set[str]) -> None:
"""`rebuild_dupes`' hand-over to a group's kept role: the earliest
`read_at` of `sources` if it has none, and each agent's latest vote among
them unless it has one of its own."""
wanted = json.dumps(sorted(sources))
conn.execute(
"""UPDATE roles SET read_at = (SELECT MIN(read_at) FROM roles
WHERE uid IN (SELECT value FROM json_each(?)))
WHERE uid = ? AND read_at IS NULL""", (wanted, kept))
voted = {r[0] for r in conn.execute("SELECT agent_id FROM role_votes WHERE uid = ?", (kept,))}
moves: dict[int, str] = {}
for agent_id, uid in conn.execute(
"""SELECT agent_id, uid FROM role_votes WHERE uid IN (SELECT value FROM json_each(?))
ORDER BY voted_at, uid""", (wanted,)):
if agent_id not in voted:
moves[agent_id] = uid # the latest vote wins, by the order above
conn.executemany("UPDATE role_votes SET uid = ? WHERE agent_id = ? AND uid = ?",
[(kept, agent_id, uid) for agent_id, uid in moves.items()])
class _StateMap(MutableMapping[str, str]):
"""`Store.state`: the key/value table, read and written one key at a time."""
@@ -443,8 +532,39 @@ class Store:
return None
role = ScoredRole.from_row(row)
role.tags = self._tags_of(uid)
role.copies = self._copies_of(uid)
return role
def _copies_of(self, uid: str) -> list[RoleCopy]:
"""The open copies of one kept role, oldest first; empty for a copy
or a role without any. See `Role.copies`."""
return [RoleCopy.from_row(r) for r in self._rows(
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))]
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
"""`uid -> open copies` for a page of kept roles, one grouped query
over `idx_roles_dup_of`. A role without copies is absent. The index
is named because the planner otherwise takes `idx_roles_closed` for
`closed_at IS NULL` and reads every open role, 19 ms against 0.1 on
the live database."""
if not uids:
return {}
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
"GROUP BY dup_of", (json.dumps(list(uids)),))}
def rebuild_dupes(self) -> DupeCounts:
"""Recompute which open roles are copies of one posting, in one
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
because it reads before it writes: a deferred transaction that a
pull's upsert wrote under in between could not take the write lock
and would fail rather than wait."""
with self.transaction() as conn:
conn.execute("BEGIN IMMEDIATE")
return rebuild_dupes(conn)
def set_read(self, uid: str, read: bool = True) -> Role:
"""Mark a role read, or unread again. Stamps `read_at` only the first
time — a role already read keeps the moment it was first opened —
@@ -677,9 +797,11 @@ class Store:
row = dict(r)
row["excerpt"] = excerpt(row["excerpt"])
found.append(RoleRow.from_row(row))
by_uid = self._tags_by_uid([r.uid for r in found])
for r in found:
r.tags = by_uid.get(r.uid, [])
by_uid = self._tags_by_uid([f.uid for f in found])
copies = self._copy_counts([f.uid for f in found])
for f in found:
f.tags = by_uid.get(f.uid, [])
f.copies = copies.get(f.uid, 0)
return found
def count_roles(self, company: str | None = None, place: str | None = None,
@@ -749,8 +871,10 @@ class Store:
pay_unknown: bool = False) -> tuple[str, list[Any]]:
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
its parameters in order. The agent's votes are always joined as `v`
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
where: list[str] = []
(see `_WITH_VOTES`); its scores as `s` only when ordering by them. A
copy of another role (`rebuild_dupes`) is never listed: its kept role
stands for it and counts its copies."""
where: list[str] = ["roles.dup_of IS NULL"]
params: list[Any] = []
if company:
where.append("company LIKE ? ESCAPE '\\'")
@@ -831,7 +955,7 @@ class Store:
first and then alphabetically. What `/api/filters` serves; the
companies `agent_id` hid are left out unless `show_muted`, the same
default as `roles`, and with no agent nothing is hidden."""
where = "r.closed_at IS NULL"
where = "r.closed_at IS NULL AND r.dup_of IS NULL"
params: list[Any] = []
if not show_muted and agent_id is not None:
where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
@@ -913,6 +1037,7 @@ class Store:
WHERE p.agent_id = a.id
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
AND top.dup_of IS NULL
WHERE s.gated_reason = '')
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
@@ -924,7 +1049,8 @@ class Store:
batch asks again when it runs and must finish what it was dealt;
otherwise only the roles in `_top_scope`."""
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")]
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
.replace("dup_of", "r.dup_of")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
@@ -1033,6 +1159,7 @@ class Store:
top, params = self._top_scope("summarise")
where += [
"r.closed_at IS NULL",
"r.dup_of IS NULL",
f"""NOT EXISTS (SELECT 1 FROM state
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
@@ -1106,11 +1233,12 @@ class Store:
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
"""Open roles with no vector from `model`, newest first, so a backlog
embedded from the top improves what is on screen first."""
embedded from the top improves what is on screen first. A copy is
not embedded: it is never scored."""
return [Role.from_row(r) for r in self._rows(
"""SELECT r.* FROM roles r
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
WHERE v.uid IS NULL AND r.closed_at IS NULL
WHERE v.uid IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
@@ -1239,10 +1367,11 @@ class Store:
# --------------------------------------------------------------- scores
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
"""Every open role of a company this agent has not hidden, with the
agent's affinity for the company and how the role meets each gate: 1
match, 0 no match, -1 the column is empty. The gates are evaluated
here, in SQL, so a `require` gate is the `WHERE` it reads as, written
"""Every open role of a company this agent has not hidden, copies
(`rebuild_dupes`) aside, with the agent's affinity for the company
and how the role meets each gate: 1 match, 0 no match, -1 the column
is empty. The gates are evaluated here, in SQL, so a `require` gate is
the `WHERE` it reads as, written
as a column so a gated role keeps its row and reason."""
columns: list[str] = []
params: list[Any] = []
@@ -1257,7 +1386,8 @@ class Store:
f"""SELECT {select} FROM roles r
LEFT JOIN agent_companies ac
ON ac.company_key = r.company_key AND ac.agent_id = ?
WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""",
WHERE r.closed_at IS NULL AND r.dup_of IS NULL
AND COALESCE(ac.affinity, 0) <> -1""",
[*params, agent_id])
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
affinity=int(r["affinity"]),
@@ -1403,10 +1533,11 @@ class Store:
def open_role_counts(self) -> dict[str, int]:
"""`company_key -> open role count`, for the company list's row
count. A key with no open roles is simply absent."""
count. A key with no open roles is simply absent. Copies are not
counted, so the number is the rows the feed lists for the company."""
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
"SELECT company_key, COUNT(*) AS n FROM roles "
"WHERE closed_at IS NULL GROUP BY company_key")}
"WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")}
def open_companies(self, agent_id: int | None = None
) -> list[tuple[str, list[str], int, int]]:
@@ -1415,7 +1546,7 @@ class Store:
company row's name and then every spelling its roles carry, commonest
first, so a search finds a company by any of them. A company
`agent_id` hid is left out; `affinity` is that agent's, 0 without
one."""
one. Copies are not counted, as in the feed."""
rows = self._rows(
"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n,
c.name AS own, COALESCE(ac.affinity, 0) AS affinity
@@ -1423,7 +1554,7 @@ class Store:
LEFT JOIN companies c ON c.key = r.company_key
LEFT JOIN agent_companies ac ON ac.company_key = r.company_key
AND ac.agent_id = ?
WHERE r.closed_at IS NULL AND r.company_key != ''
WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != ''
AND COALESCE(ac.affinity, 0) != -1
GROUP BY r.company_key, r.company
ORDER BY r.company_key, n DESC, r.company""", (agent_id,))
@@ -1439,13 +1570,16 @@ class Store:
def open_places(self) -> list[tuple[str, str, int]]:
"""`(place, country, open roles)` for every place line an open role
carries, what `places.suggest` reads."""
carries, what `places.suggest` reads. Copies are not counted, as in
the feed."""
return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows(
"SELECT place, country, COUNT(*) AS n FROM roles "
"WHERE closed_at IS NULL AND place != '' GROUP BY place, country")]
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
"GROUP BY place, country")]
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
"""Per subscription id, its open roles counted by `country`."""
"""Per subscription id, its open roles counted by `country`, copies
included: what a board or a search holds, not what the feed lists."""
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
GROUP BY subscription_id, country""")
+7 -2
View File
@@ -133,7 +133,9 @@ def role_facets(store: StoreDep, filters: FeedFilters) -> Facets:
def get_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole:
"""One role, open or closed, for the read page, with `agent`'s vote and
score, and the prompt that gave its fit (the default agent's when absent;
None when there is no agent)."""
None when there is no agent). `copies` lists the open copies of a kept
role (`RoleCopy`, for "Also posted"); a copy itself opens too, with
`dup_of` naming its kept role."""
role = store.role(uid, _agent_id(store, agent))
if role is None:
raise HTTPException(404, f"no such role: {uid}")
@@ -226,7 +228,10 @@ def list_roles(
`region:skane`, `place:malmo`, matched on whole folded words) and
`seniority` (repeated levels), each several values joined by OR. A role
with no level passes the seniority filter unless `seniority_unknown` is
false. The parameters are read by `_feed_filters`."""
false. The parameters are read by `_feed_filters`.
A copy of another role (`Store.rebuild_dupes`) is never a row; its kept
role's `copies` says how many it stands for."""
return RolesPage(items=store.roles_page(limit=limit, offset=offset, **filters),
total=store.count_roles(**filters))
+177
View File
@@ -0,0 +1,177 @@
"""`roles.dup_of`: migration 18, `Store.rebuild_dupes`, and what a copy is left out of."""
from __future__ import annotations
import sqlite3
from datetime import UTC, datetime
from pathlib import Path
import numpy as np
import pytest
from desk.models import Item
from app.storage import Store
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def role(source_id: str, *, company: str = "SAP", title: str = "Forward Deployed Engineer",
place: str = "Garching", body: str = "The posting.", day: int = 20,
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://jobs.example/{source_id}",
title=title, body=body, published_at=datetime(2026, 9, day, tzinfo=UTC),
raw={"company": company, "place": place, "board": "successfactors", **raw})
item.source = "ats"
return item
def dup_of(store: Store) -> dict[str, str | None]:
with sqlite3.connect(store.path) as conn:
return dict(conn.execute("SELECT uid, dup_of FROM roles"))
def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart(
store: Store) -> None:
"""SAP's repeats: three Garching rows are one kept role (the one with
the description) and two copies; New York is its own role."""
garching = [role("1", body=""), role("2"), role("3", body="")]
york = role("4", place="New York, NY, US")
for item in (*garching, york):
store.upsert_role(item)
assert store.rebuild_dupes() == (1, 2, 2)
kept = garching[1].uid
assert dup_of(store) == {garching[0].uid: kept, kept: None,
garching[2].uid: kept, york.uid: None}
assert store.rebuild_dupes().changed == 0 # stable between runs
def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store) -> None:
board = role("board", company="Trackunit", title="Backend Engineer", place="Aalborg",
board="teamtailor", body="")
search = role("search", company="Trackunit", title="Backend Engineer",
place="Aalborg, Denmark", board=None, provider="jobindex")
for item in (search, board):
store.upsert_role(item)
store.rebuild_dupes()
assert dup_of(store) == {board.uid: None, search.uid: board.uid}
def test_berlin_and_munich_stay_two_roles(store: Store) -> None:
for item in (role("b", place="Berlin"), role("m", place="Munich")):
store.upsert_role(item)
assert store.rebuild_dupes() == (0, 0, 0)
def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared(
store: Store) -> None:
first, second, third = role("1", day=18), role("2", day=19), role("3", day=20)
for item in (first, second, third):
store.upsert_role(item)
store.rebuild_dupes()
assert dup_of(store)[second.uid] == first.uid
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,))
store.rebuild_dupes()
assert dup_of(store) == {first.uid: None, second.uid: None, third.uid: second.uid}
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-26' WHERE uid = ?", (third.uid,))
store.rebuild_dupes()
assert set(dup_of(store).values()) == {None}
def test_read_and_votes_move_to_the_kept_role(store: Store) -> None:
"""A copy read and liked before it was known as one: the kept role
becomes read and liked. An agent that voted on the kept role keeps its
own vote, and a later unread on the kept role is not undone."""
one, two = store.save_agent("One"), store.save_agent("Two")
kept, copy = role("kept", day=18), role("copy", day=19)
store.upsert_role(copy)
store.set_vote(one.id, copy.uid, 1)
store.set_vote(two.id, copy.uid, -1)
store.upsert_role(kept)
store.set_vote(two.id, kept.uid, 0)
store.rebuild_dupes()
read = store.role(copy.uid).read_at # type: ignore[union-attr]
assert store.role(kept.uid).read_at == read # type: ignore[union-attr]
assert store.votes(one.id) == {kept.uid: 1}
assert store.votes(two.id) == {kept.uid: 0, copy.uid: -1}
store.set_read(kept.uid, False)
store.rebuild_dupes()
assert store.role(kept.uid).read_at is None # type: ignore[union-attr]
def test_a_closed_kept_role_hands_its_read_and_vote_to_the_next(store: Store) -> None:
agent = store.save_agent("One")
first, second = role("1", day=18), role("2", day=19)
for item in (first, second):
store.upsert_role(item)
store.rebuild_dupes()
store.set_vote(agent.id, first.uid, 1)
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,))
# A third copy keeps the group alive once the first has closed.
store.upsert_role(role("3", day=20))
store.rebuild_dupes()
assert store.votes(agent.id) == {second.uid: 1}
assert store.role(second.uid).read_at is not None # type: ignore[union-attr]
def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None:
kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin")
for item in (kept, copy, other):
store.upsert_role(item)
store.set_tags(item.uid, ["python"])
store.rebuild_dupes()
agent = store.save_agent("A", intent="engineering")
assert {r.uid for r in store.roles()} == {kept.uid, other.uid}
rows = {r.uid: r.copies for r in store.roles_page()}
assert rows == {kept.uid: 1, other.uid: 0}
assert store.count_roles() == 2
assert sum(n for _, n in store.role_facets()["seniority"]) == 2
assert store.tag_counts() == [("python", 2, 2)]
assert {c.uid for c in store.score_candidates(agent.id, [])} == {kept.uid, other.uid}
assert {r.uid for r in store.roles_to_embed("m", 10)} == {kept.uid, other.uid}
assert store.open_role_counts() == {"sap": 2}
assert [count for _, _, count, _ in store.open_companies()] == [2]
store.replace_scores(agent.id, [(u, 0.9, "") for u in (kept.uid, copy.uid, other.uid)])
assert {r.uid for r in store.extract_waiting(10)} == {kept.uid, other.uid}
assert {r.uid for r in store.summarise_waiting(10)} == {kept.uid, other.uid}
# The copy still opens by uid, and the kept role lists it.
assert store.role(copy.uid).dup_of == kept.uid # type: ignore[union-attr]
assert [c.uid for c in store.role(kept.uid).copies] == [copy.uid] # type: ignore[union-attr]
assert store.role(other.uid).copies == [] # type: ignore[union-attr]
def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
path = tmp_path / "v17.db"
store = Store(path)
first, second = role("1", day=18), role("2", day=19)
for item in (first, second):
store.upsert_role(item)
store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))])
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
conn.execute("PRAGMA user_version = 17")
migrated = Store(path)
assert migrated.migrations_applied == [18]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
).fetchone() is not None
+34 -5
View File
@@ -31,11 +31,11 @@ class FakeConfig:
searches: ClassVar[list[object]] = []
def _item(board: str, slug: str, job_id: str) -> Item:
def _item(board: str, slug: str, job_id: str, company: str = "Acme") -> Item:
item = Item(
source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}",
title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "place": "Berlin", "department": "",
raw={"company": company, "place": "Berlin", "department": "",
"function": "engineering", "remote": None, "salary_text": "",
"board": board, "slug": slug})
item.source = "ats"
@@ -126,7 +126,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
store.upsert_role(_item("greenhouse", "acme", "2"), sub)
store.upsert_role(_item("greenhouse", "other", "9"), other)
store.upsert_role(_item("greenhouse", "other", "9", company="Other"), other)
# acme answered this run and now lists only "1"; other's board failed
# (or its batch never got to it) and stamped nothing.
@@ -140,7 +140,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 1, "closed": 1}
"roles": 1, "closed": 1, "copies": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -161,10 +161,39 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 0, "closed": 0}
"roles": 0, "closed": 0, "copies": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
"""The settle's last step: a board's repeats of one job are one role,
and when the kept one closes the next copy is kept in its place."""
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
company="Acme", board="greenhouse")
for job_id in ("1", "2", "3"):
store.upsert_role(_item("greenhouse", "acme", job_id), sub)
store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:complete:greenhouse:acme"] = "1"
store.state["ats:ids:greenhouse:acme"] = json.dumps(
["greenhouse:acme:1", "greenhouse:acme:2", "greenhouse:acme:3"])
class FakeTally:
results = ({"roles": 3, "run": "run-1"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result["copies"] == 2
[kept] = store.roles_page()
assert kept.copies == 2
gone = store.role(kept.uid).source_id # type: ignore[union-attr]
store.state["ats:ids:greenhouse:acme"] = json.dumps(
[f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone])
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert (result["closed"], result["copies"]) == (1, 1)
[after] = store.roles_page()
assert after.uid != kept.uid and after.copies == 1
# ---------------------------------------------- whole pulls, fetch stubbed
def _row(job_id: str) -> dict[str, object]:
+12 -5
View File
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 17
assert LATEST == 18
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -88,8 +88,15 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
def _undo_v18(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v17 shape: no copies."""
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
def _undo_v17(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v16 shape: no folded place on a role."""
_undo_v18(conn)
conn.execute("ALTER TABLE roles DROP COLUMN place_fold")
@@ -159,7 +166,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17]
assert migrated.migrations_applied == [15, 16, 17, 18]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -364,7 +371,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17]
assert migrated.migrations_applied == [14, 15, 16, 17, 18]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -388,7 +395,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17]
assert migrated.migrations_applied == [16, 17, 18]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -417,7 +424,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17]
assert migrated.migrations_applied == [17, 18]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
+27 -1
View File
@@ -1159,7 +1159,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
"seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state",
"original_url", "posted_at", "score", "gated_reason", "prompt", "vote",
"voted_at", "read_at", "tags", "excerpt",
"voted_at", "read_at", "tags", "excerpt", "copies",
}
assert row["uid"] == uid
assert row["company"] == "Acme" and row["place"] == "Berlin, Germany"
@@ -1644,3 +1644,29 @@ def test_summarise_queues_one_role_and_the_job_writes_it(
assert asked == [[uid]]
assert store.role(uid).summary == "Backend role in Berlin, Python." # type: ignore[union-attr]
assert client.post(f"/api/roles/{uid}/summarise").status_code == 409
def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
client: TestClient, store: Store) -> None:
"""The contract the reader and the feed row build on: `copies` on a row
is a count, on `GET /api/roles/{uid}` the list of open copies, and a copy
still opens, with `dup_of`."""
board = posting("board", company="Acme", place="Ballerup", board="greenhouse")
board.body = "The whole posting."
search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex")
for item in (board, search, posting("alone", company="Other")):
store.upsert_role(item)
store.rebuild_dupes()
page = client.get("/api/roles").json()
assert page["total"] == 2
assert {r["uid"]: r["copies"] for r in page["items"]} == {
board.uid: 1, posting("alone").uid: 0}
kept = client.get(f"/api/roles/{board.uid}").json()
assert kept["dup_of"] is None
assert kept["copies"] == [{
"uid": search.uid, "host": "jobindex", "place": "Ballerup, Denmark",
"original_url": search.location, "posted_at": kept["copies"][0]["posted_at"]}]
copy = client.get(f"/api/roles/{search.uid}").json()
assert copy["dup_of"] == board.uid and copy["copies"] == []