diff --git a/app/cli.py b/app/cli.py index fe92c5d..de75961 100644 --- a/app/cli.py +++ b/app/cli.py @@ -7,6 +7,7 @@ app searches add|list|remove app agents list|prompt app stages [extract|summarise on|off] + app dupes app db init `app job` queues the work on a queue of its own and drains it here, so a @@ -155,6 +156,8 @@ def build_parser() -> argparse.ArgumentParser: stages.add_argument("name", nargs="?", choices=["extract", "summarise"]) stages.add_argument("state", nargs="?", choices=["on", "off"]) + sub.add_parser("dupes", help="recompute which open roles are copies of one posting") + db = sub.add_parser("db", help="database housekeeping") db.add_argument("action", choices=["init"]) return parser @@ -179,6 +182,8 @@ def main(argv: list[str] | None = None) -> int: return run_agents(args) if args.command == "stages": return run_stages(args) + if args.command == "dupes": + return run_dupes(args) return init_db(args) except (FileNotFoundError, config_mod.ConfigError) as exc: print(exc, file=sys.stderr) @@ -626,6 +631,18 @@ def run_stages(args: argparse.Namespace) -> int: return 0 +def run_dupes(args: argparse.Namespace) -> int: + """`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`, + now. Prints the groups (kept roles with a copy), the copies the feed + hides and the rows whose `dup_of` changed, as JSON.""" + from app.storage import Store + + config = config_mod.load(args.config) + counts = Store(config.database).rebuild_dupes() + print(json.dumps(counts._asdict())) + return 0 + + def init_db(args: argparse.Namespace) -> int: """Create the file, or bring it to the current schema.""" from app.storage import LATEST, Store diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 45aa04d..46a0588 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -219,6 +219,11 @@ def close_boards(config: Config, store: Store, tally: Tally, *, A board or search switched off while its batch was in flight had its roles reopened by that batch, after the switch closed them, so every switched-off one is closed again here. + + After the closes, `Store.rebuild_dupes` recomputes which open roles are + copies of one posting, so a closed kept role hands its group on and the + roles this run added join theirs; `copies` is how many open roles it + leaves hidden as copies. """ answered = searched = failed = incomplete = closed = 0 stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds") @@ -252,9 +257,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *, closed += store.close_unseen("search", stale) if run_marker: release_run(store, run_marker) + copies = store.rebuild_dupes().copies roles = sum(int(result.get("roles", 0)) for result in tally.results) return {"boards": answered, "searches": searched, "failed": failed, - "incomplete": incomplete, "roles": roles, "closed": closed} + "incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies} def failing_boards(store: Store) -> dict[str, tuple[str, str]]: diff --git a/app/models.py b/app/models.py index 84e72b4..2af2065 100644 --- a/app/models.py +++ b/app/models.py @@ -763,6 +763,18 @@ JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))] # ------------------------------------------------------------------ models +class RoleCopy(Model): + """One open copy of a kept role, as the reader lists it under "Also + posted": where it is listed (`host`, the search provider or the board's + ATS) and the link that opens it there.""" + + uid: str + host: str = "" + place: str = "" + original_url: str = "" + posted_at: str = "" + + class Role(Model): """A row of `roles`: one posting on one board.""" @@ -819,6 +831,13 @@ class Role(Model): #: The open vocabulary `summarise` read off the posting, sorted. Not a #: column: loaded from `role_tags` alongside the row, see `Store.role`. tags: list[str] = [] + #: The uid of the kept role when this one is a copy of the same posting + #: (`app.dupes`, `Store.rebuild_dupes`), else None. A copy is left out of + #: the feed and the counts but still opens by uid. + dup_of: str | None = None + #: The open copies of this role, oldest first, when it is the kept one; + #: empty otherwise. Not a column: filled by `Store.role`. + copies: list[RoleCopy] = [] @computed_field # type: ignore[prop-decorator] @property @@ -895,7 +914,8 @@ class RoleRow(Model): derived as on `Role`. `original_url` is one short string, kept so the `o` key opens a selected row's posting without opening the reader. `seniority` is the title's level until extract reads the posting, '' - when neither says.""" + when neither says. `copies` counts the open copies the feed hides behind + this row.""" uid: str company_key: str = "" @@ -920,6 +940,8 @@ class RoleRow(Model): read_at: str | None = None tags: list[str] = [] excerpt: str = "" + #: How many open roles are copies of this one (`Role.copies`), 0 when none. + copies: int = 0 @computed_field # type: ignore[prop-decorator] @property diff --git a/app/storage/schema.py b/app/storage/schema.py index 662b3b5..0dd9dee 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -92,6 +92,13 @@ CREATE TABLE IF NOT EXISTS roles ( -- whose description is only a teaser. article_text TEXT NOT NULL DEFAULT '', article_at TEXT, + -- The uid of the kept role when this open role is a copy of one posting + -- (app/dupes.py), NULL on the kept role, on a role without copies and on + -- every closed role. Written only by Store.rebuild_dupes, at the end of + -- every pull; the feed, the counts and the model stages skip a copy. + -- Not the last column: SQLite cannot drop a last column that a comment + -- precedes, which the migration tests do. + dup_of TEXT, created_at TEXT NOT NULL, salary_text TEXT NOT NULL DEFAULT '' ); @@ -292,6 +299,10 @@ CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC); CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote); CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model); CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag); +-- The copies of a kept role, for `Store.role` and the feed's copy counts. +-- Partial: all but about 3% of roles are no copy. +CREATE INDEX IF NOT EXISTS idx_roles_dup_of + ON roles(dup_of) WHERE dup_of IS NOT NULL; """ #: The table whose absence marks a file that has never been a roledesk @@ -455,6 +466,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; # A Python step: the fold and the title classifier are Python, as in 11. (17, "roles.place_fold for the whole-word place filter, and roles.seniority from " "the title where extract has not read the role", lambda conn: _fold_places(conn)), + # A Python step: the first rebuild runs the rule in app/dupes.py. + (18, "roles.dup_of: one posting listed several times is one role, the others " + "copies of it", lambda conn: _add_dup_of(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -566,6 +580,18 @@ def _fold_places(conn: sqlite3.Connection) -> None: len(folds), len(place_updates), len(level_updates)) +def _add_dup_of(conn: sqlite3.Connection) -> None: + """Migration 18's step. Adds `roles.dup_of` and runs the first rebuild + (`store.rebuild_dupes`) inside `_apply`'s transaction, so a migrated file + hides its copies from the first open rather than from the next pull. + Imported here, not at the top: `store` imports this module.""" + from app.storage.store import rebuild_dupes + + conn.execute("ALTER TABLE roles ADD COLUMN dup_of TEXT") + counts = rebuild_dupes(conn) + log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/storage/store.py b/app/storage/store.py index a8dc590..dad2789 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -14,16 +14,17 @@ from collections.abc import Callable, Iterator, MutableMapping, Sequence from contextlib import closing, contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, NamedTuple from uuid import uuid4 import numpy as np from desk.models import Item -from app import places +from app import dupes, places from app.models import ( AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application, - ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, + ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow, + ScoredRole, Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key, ) @@ -50,8 +51,9 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit") #: never answers is otherwise paid for on every pull for as long as it exists. GIVE_UP_AFTER = 3 -#: A role `extract` has yet to read: never answered, still open, not given up. -_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL " +#: A role `extract` has yet to read: never answered, still open, no copy of +#: another (`rebuild_dupes`), not given up. +_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL AND dup_of IS NULL " f"AND extract_tries < {GIVE_UP_AFTER}") #: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged. @@ -226,6 +228,93 @@ def _int_or_none(value: Any) -> int | None: return None +#: The host `app.dupes.Copy.host` and `RoleCopy.host` read: the provider of +#: a search's row, else the ATS a board runs on, as `posted_host` reads it. +_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')" + + +class DupeCounts(NamedTuple): + """What `rebuild_dupes` left and did: kept roles with at least one copy, + open roles that are a copy, and rows whose `dup_of` it rewrote.""" + + groups: int + copies: int + changed: int + + +def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts: + """Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside + the caller's transaction: `Store.rebuild_dupes` and migration 18. + + Only the open roles of a `(company_key, title_key)` with more than one + open role can be a copy, a few thousand of the ~34k, so only those are + read. A row whose `dup_of` differs from the rule's answer is rewritten, + and a `dup_of` left on a closed role, or on one no longer in such a pair, + is cleared: a copy never outlives its posting, and a closed kept role + hands its group to the next copy here. + + Where a group changed (a member's `dup_of` was rewritten), its kept role + takes over what was done to the posting under another uid. The sources + are the group's members and the roles they were copies of before, so a + closed kept role still hands over. The kept role takes the earliest + `read_at` of them when it has none, and each agent's latest vote among + them moves to it unless that agent already voted on it; a vote that + cannot move stays where it is. An unchanged group is not touched, so + marking a kept role unread again sticks. Applications keep their + `role_uid`, and scores of a copy go at the agent's next `score`.""" + rows = conn.execute( + f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host, + description != '' AS described, posted_at + FROM roles + WHERE closed_at IS NULL AND company_key != '' + AND (company_key, title_key) IN ( + SELECT company_key, title_key FROM roles + WHERE closed_at IS NULL AND company_key != '' + GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall() + new = dupes.assign( + dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4], + host=r[5], has_description=bool(r[6]), posted_at=r[7] or None) + for r in rows) + old: dict[str, str | None] = dict.fromkeys(new) + old.update((r[0], r[1]) for r in conn.execute( + "SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL")) + updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was] + conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates) + + changed = {uid for _, uid in updates} + members: dict[str, set[str]] = {} + for uid, dup in new.items(): + members.setdefault(dup or uid, set()).add(uid) + groups = {kept: group for kept, group in members.items() if len(group) > 1} + for kept, group in groups.items(): + if not group & changed: + continue + sources = (group | {old.get(uid) or uid for uid in group}) - {kept} + _carry(conn, kept, sources) + return DupeCounts(groups=len(groups), copies=sum(dup is not None for dup in new.values()), + changed=len(updates)) + + +def _carry(conn: sqlite3.Connection, kept: str, sources: set[str]) -> None: + """`rebuild_dupes`' hand-over to a group's kept role: the earliest + `read_at` of `sources` if it has none, and each agent's latest vote among + them unless it has one of its own.""" + wanted = json.dumps(sorted(sources)) + conn.execute( + """UPDATE roles SET read_at = (SELECT MIN(read_at) FROM roles + WHERE uid IN (SELECT value FROM json_each(?))) + WHERE uid = ? AND read_at IS NULL""", (wanted, kept)) + voted = {r[0] for r in conn.execute("SELECT agent_id FROM role_votes WHERE uid = ?", (kept,))} + moves: dict[int, str] = {} + for agent_id, uid in conn.execute( + """SELECT agent_id, uid FROM role_votes WHERE uid IN (SELECT value FROM json_each(?)) + ORDER BY voted_at, uid""", (wanted,)): + if agent_id not in voted: + moves[agent_id] = uid # the latest vote wins, by the order above + conn.executemany("UPDATE role_votes SET uid = ? WHERE agent_id = ? AND uid = ?", + [(kept, agent_id, uid) for agent_id, uid in moves.items()]) + + class _StateMap(MutableMapping[str, str]): """`Store.state`: the key/value table, read and written one key at a time.""" @@ -443,8 +532,39 @@ class Store: return None role = ScoredRole.from_row(row) role.tags = self._tags_of(uid) + role.copies = self._copies_of(uid) return role + def _copies_of(self, uid: str) -> list[RoleCopy]: + """The open copies of one kept role, oldest first; empty for a copy + or a role without any. See `Role.copies`.""" + return [RoleCopy.from_row(r) for r in self._rows( + f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles + WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))] + + def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]: + """`uid -> open copies` for a page of kept roles, one grouped query + over `idx_roles_dup_of`. A role without copies is absent. The index + is named because the planner otherwise takes `idx_roles_closed` for + `closed_at IS NULL` and reads every open role, 19 ms against 0.1 on + the live database.""" + if not uids: + return {} + return {str(r["dup_of"]): int(r["n"]) for r in self._rows( + "SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of " + "WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL " + "GROUP BY dup_of", (json.dumps(list(uids)),))} + + def rebuild_dupes(self) -> DupeCounts: + """Recompute which open roles are copies of one posting, in one + transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE` + because it reads before it writes: a deferred transaction that a + pull's upsert wrote under in between could not take the write lock + and would fail rather than wait.""" + with self.transaction() as conn: + conn.execute("BEGIN IMMEDIATE") + return rebuild_dupes(conn) + def set_read(self, uid: str, read: bool = True) -> Role: """Mark a role read, or unread again. Stamps `read_at` only the first time — a role already read keeps the moment it was first opened — @@ -677,9 +797,11 @@ class Store: row = dict(r) row["excerpt"] = excerpt(row["excerpt"]) found.append(RoleRow.from_row(row)) - by_uid = self._tags_by_uid([r.uid for r in found]) - for r in found: - r.tags = by_uid.get(r.uid, []) + by_uid = self._tags_by_uid([f.uid for f in found]) + copies = self._copy_counts([f.uid for f in found]) + for f in found: + f.tags = by_uid.get(f.uid, []) + f.copies = copies.get(f.uid, 0) return found def count_roles(self, company: str | None = None, place: str | None = None, @@ -749,8 +871,10 @@ class Store: pay_unknown: bool = False) -> tuple[str, list[Any]]: """The `FROM ... WHERE ...` that `roles` and `count_roles` share, with its parameters in order. The agent's votes are always joined as `v` - (see `_WITH_VOTES`); its scores as `s` only when ordering by them.""" - where: list[str] = [] + (see `_WITH_VOTES`); its scores as `s` only when ordering by them. A + copy of another role (`rebuild_dupes`) is never listed: its kept role + stands for it and counts its copies.""" + where: list[str] = ["roles.dup_of IS NULL"] params: list[Any] = [] if company: where.append("company LIKE ? ESCAPE '\\'") @@ -831,7 +955,7 @@ class Store: first and then alphabetically. What `/api/filters` serves; the companies `agent_id` hid are left out unless `show_muted`, the same default as `roles`, and with no agent nothing is hidden.""" - where = "r.closed_at IS NULL" + where = "r.closed_at IS NULL AND r.dup_of IS NULL" params: list[Any] = [] if not show_muted and agent_id is not None: where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies " @@ -913,6 +1037,7 @@ class Store: WHERE p.agent_id = a.id AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '') JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL + AND top.dup_of IS NULL WHERE s.gated_reason = '') WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?) AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""", @@ -924,7 +1049,8 @@ class Store: batch asks again when it runs and must finish what it was dealt; otherwise only the roles in `_top_scope`.""" where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at") - .replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")] + .replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries") + .replace("dup_of", "r.dup_of")] if uids is not None: where.append("r.uid IN (SELECT value FROM json_each(?))") return " AND ".join(where), [json.dumps(list(uids))] @@ -1033,6 +1159,7 @@ class Store: top, params = self._top_scope("summarise") where += [ "r.closed_at IS NULL", + "r.dup_of IS NULL", f"""NOT EXISTS (SELECT 1 FROM state WHERE key = '{self._SUMMARY_MISSED}' || r.uid AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""", @@ -1106,11 +1233,12 @@ class Store: def roles_to_embed(self, model: str, limit: int) -> list[Role]: """Open roles with no vector from `model`, newest first, so a backlog - embedded from the top improves what is on screen first.""" + embedded from the top improves what is on screen first. A copy is + not embedded: it is never scored.""" return [Role.from_row(r) for r in self._rows( """SELECT r.* FROM roles r LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ? - WHERE v.uid IS NULL AND r.closed_at IS NULL + WHERE v.uid IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))] def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int: @@ -1239,10 +1367,11 @@ class Store: # --------------------------------------------------------------- scores def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]: - """Every open role of a company this agent has not hidden, with the - agent's affinity for the company and how the role meets each gate: 1 - match, 0 no match, -1 the column is empty. The gates are evaluated - here, in SQL, so a `require` gate is the `WHERE` it reads as, written + """Every open role of a company this agent has not hidden, copies + (`rebuild_dupes`) aside, with the agent's affinity for the company + and how the role meets each gate: 1 match, 0 no match, -1 the column + is empty. The gates are evaluated here, in SQL, so a `require` gate is + the `WHERE` it reads as, written as a column so a gated role keeps its row and reason.""" columns: list[str] = [] params: list[Any] = [] @@ -1257,7 +1386,8 @@ class Store: f"""SELECT {select} FROM roles r LEFT JOIN agent_companies ac ON ac.company_key = r.company_key AND ac.agent_id = ? - WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""", + WHERE r.closed_at IS NULL AND r.dup_of IS NULL + AND COALESCE(ac.affinity, 0) <> -1""", [*params, agent_id]) return [Candidate(uid=r["uid"], posted_at=r["posted_at"], affinity=int(r["affinity"]), @@ -1403,10 +1533,11 @@ class Store: def open_role_counts(self) -> dict[str, int]: """`company_key -> open role count`, for the company list's row - count. A key with no open roles is simply absent.""" + count. A key with no open roles is simply absent. Copies are not + counted, so the number is the rows the feed lists for the company.""" return {str(r["company_key"]): int(r["n"]) for r in self._rows( "SELECT company_key, COUNT(*) AS n FROM roles " - "WHERE closed_at IS NULL GROUP BY company_key")} + "WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")} def open_companies(self, agent_id: int | None = None ) -> list[tuple[str, list[str], int, int]]: @@ -1415,7 +1546,7 @@ class Store: company row's name and then every spelling its roles carry, commonest first, so a search finds a company by any of them. A company `agent_id` hid is left out; `affinity` is that agent's, 0 without - one.""" + one. Copies are not counted, as in the feed.""" rows = self._rows( """SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n, c.name AS own, COALESCE(ac.affinity, 0) AS affinity @@ -1423,7 +1554,7 @@ class Store: LEFT JOIN companies c ON c.key = r.company_key LEFT JOIN agent_companies ac ON ac.company_key = r.company_key AND ac.agent_id = ? - WHERE r.closed_at IS NULL AND r.company_key != '' + WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != '' AND COALESCE(ac.affinity, 0) != -1 GROUP BY r.company_key, r.company ORDER BY r.company_key, n DESC, r.company""", (agent_id,)) @@ -1439,13 +1570,16 @@ class Store: def open_places(self) -> list[tuple[str, str, int]]: """`(place, country, open roles)` for every place line an open role - carries, what `places.suggest` reads.""" + carries, what `places.suggest` reads. Copies are not counted, as in + the feed.""" return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows( "SELECT place, country, COUNT(*) AS n FROM roles " - "WHERE closed_at IS NULL AND place != '' GROUP BY place, country")] + "WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' " + "GROUP BY place, country")] def open_countries_by_subscription(self) -> dict[int, dict[str, int]]: - """Per subscription id, its open roles counted by `country`.""" + """Per subscription id, its open roles counted by `country`, copies + included: what a board or a search holds, not what the feed lists.""" rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles WHERE closed_at IS NULL AND subscription_id IS NOT NULL GROUP BY subscription_id, country""") diff --git a/app/web/api.py b/app/web/api.py index a573b70..4afc9fd 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -133,7 +133,9 @@ def role_facets(store: StoreDep, filters: FeedFilters) -> Facets: def get_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole: """One role, open or closed, for the read page, with `agent`'s vote and score, and the prompt that gave its fit (the default agent's when absent; - None when there is no agent).""" + None when there is no agent). `copies` lists the open copies of a kept + role (`RoleCopy`, for "Also posted"); a copy itself opens too, with + `dup_of` naming its kept role.""" role = store.role(uid, _agent_id(store, agent)) if role is None: raise HTTPException(404, f"no such role: {uid}") @@ -226,7 +228,10 @@ def list_roles( `region:skane`, `place:malmo`, matched on whole folded words) and `seniority` (repeated levels), each several values joined by OR. A role with no level passes the seniority filter unless `seniority_unknown` is - false. The parameters are read by `_feed_filters`.""" + false. The parameters are read by `_feed_filters`. + + A copy of another role (`Store.rebuild_dupes`) is never a row; its kept + role's `copies` says how many it stands for.""" return RolesPage(items=store.roles_page(limit=limit, offset=offset, **filters), total=store.count_roles(**filters)) diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py new file mode 100644 index 0000000..4427076 --- /dev/null +++ b/tests/test_dupes_storage.py @@ -0,0 +1,177 @@ +"""`roles.dup_of`: migration 18, `Store.rebuild_dupes`, and what a copy is left out of.""" + +from __future__ import annotations + +import sqlite3 +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np +import pytest + +from desk.models import Item + +from app.storage import Store + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +def role(source_id: str, *, company: str = "SAP", title: str = "Forward Deployed Engineer", + place: str = "Garching", body: str = "The posting.", day: int = 20, + **raw: object) -> Item: + item = Item(source_id=source_id, location=f"https://jobs.example/{source_id}", + title=title, body=body, published_at=datetime(2026, 9, day, tzinfo=UTC), + raw={"company": company, "place": place, "board": "successfactors", **raw}) + item.source = "ats" + return item + + +def dup_of(store: Store) -> dict[str, str | None]: + with sqlite3.connect(store.path) as conn: + return dict(conn.execute("SELECT uid, dup_of FROM roles")) + + +def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart( + store: Store) -> None: + """SAP's repeats: three Garching rows are one kept role (the one with + the description) and two copies; New York is its own role.""" + garching = [role("1", body=""), role("2"), role("3", body="")] + york = role("4", place="New York, NY, US") + for item in (*garching, york): + store.upsert_role(item) + + assert store.rebuild_dupes() == (1, 2, 2) + kept = garching[1].uid + assert dup_of(store) == {garching[0].uid: kept, kept: None, + garching[2].uid: kept, york.uid: None} + assert store.rebuild_dupes().changed == 0 # stable between runs + + +def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store) -> None: + board = role("board", company="Trackunit", title="Backend Engineer", place="Aalborg", + board="teamtailor", body="") + search = role("search", company="Trackunit", title="Backend Engineer", + place="Aalborg, Denmark", board=None, provider="jobindex") + for item in (search, board): + store.upsert_role(item) + store.rebuild_dupes() + assert dup_of(store) == {board.uid: None, search.uid: board.uid} + + +def test_berlin_and_munich_stay_two_roles(store: Store) -> None: + for item in (role("b", place="Berlin"), role("m", place="Munich")): + store.upsert_role(item) + assert store.rebuild_dupes() == (0, 0, 0) + + +def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared( + store: Store) -> None: + first, second, third = role("1", day=18), role("2", day=19), role("3", day=20) + for item in (first, second, third): + store.upsert_role(item) + store.rebuild_dupes() + assert dup_of(store)[second.uid] == first.uid + + with sqlite3.connect(store.path) as conn: + conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,)) + store.rebuild_dupes() + assert dup_of(store) == {first.uid: None, second.uid: None, third.uid: second.uid} + + with sqlite3.connect(store.path) as conn: + conn.execute("UPDATE roles SET closed_at = '2026-09-26' WHERE uid = ?", (third.uid,)) + store.rebuild_dupes() + assert set(dup_of(store).values()) == {None} + + +def test_read_and_votes_move_to_the_kept_role(store: Store) -> None: + """A copy read and liked before it was known as one: the kept role + becomes read and liked. An agent that voted on the kept role keeps its + own vote, and a later unread on the kept role is not undone.""" + one, two = store.save_agent("One"), store.save_agent("Two") + kept, copy = role("kept", day=18), role("copy", day=19) + store.upsert_role(copy) + store.set_vote(one.id, copy.uid, 1) + store.set_vote(two.id, copy.uid, -1) + store.upsert_role(kept) + store.set_vote(two.id, kept.uid, 0) + + store.rebuild_dupes() + + read = store.role(copy.uid).read_at # type: ignore[union-attr] + assert store.role(kept.uid).read_at == read # type: ignore[union-attr] + assert store.votes(one.id) == {kept.uid: 1} + assert store.votes(two.id) == {kept.uid: 0, copy.uid: -1} + + store.set_read(kept.uid, False) + store.rebuild_dupes() + assert store.role(kept.uid).read_at is None # type: ignore[union-attr] + + +def test_a_closed_kept_role_hands_its_read_and_vote_to_the_next(store: Store) -> None: + agent = store.save_agent("One") + first, second = role("1", day=18), role("2", day=19) + for item in (first, second): + store.upsert_role(item) + store.rebuild_dupes() + store.set_vote(agent.id, first.uid, 1) + with sqlite3.connect(store.path) as conn: + conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,)) + # A third copy keeps the group alive once the first has closed. + store.upsert_role(role("3", day=20)) + + store.rebuild_dupes() + + assert store.votes(agent.id) == {second.uid: 1} + assert store.role(second.uid).read_at is not None # type: ignore[union-attr] + + +def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None: + kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin") + for item in (kept, copy, other): + store.upsert_role(item) + store.set_tags(item.uid, ["python"]) + store.rebuild_dupes() + agent = store.save_agent("A", intent="engineering") + + assert {r.uid for r in store.roles()} == {kept.uid, other.uid} + rows = {r.uid: r.copies for r in store.roles_page()} + assert rows == {kept.uid: 1, other.uid: 0} + assert store.count_roles() == 2 + assert sum(n for _, n in store.role_facets()["seniority"]) == 2 + assert store.tag_counts() == [("python", 2, 2)] + assert {c.uid for c in store.score_candidates(agent.id, [])} == {kept.uid, other.uid} + assert {r.uid for r in store.roles_to_embed("m", 10)} == {kept.uid, other.uid} + assert store.open_role_counts() == {"sap": 2} + assert [count for _, _, count, _ in store.open_companies()] == [2] + store.replace_scores(agent.id, [(u, 0.9, "") for u in (kept.uid, copy.uid, other.uid)]) + assert {r.uid for r in store.extract_waiting(10)} == {kept.uid, other.uid} + assert {r.uid for r in store.summarise_waiting(10)} == {kept.uid, other.uid} + + # The copy still opens by uid, and the kept role lists it. + assert store.role(copy.uid).dup_of == kept.uid # type: ignore[union-attr] + assert [c.uid for c in store.role(kept.uid).copies] == [copy.uid] # type: ignore[union-attr] + assert store.role(other.uid).copies == [] # type: ignore[union-attr] + + +def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None: + path = tmp_path / "v17.db" + store = Store(path) + first, second = role("1", day=18), role("2", day=19) + for item in (first, second): + store.upsert_role(item) + store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))]) + with sqlite3.connect(path) as conn: + conn.execute("DROP INDEX idx_roles_dup_of") + conn.execute("ALTER TABLE roles DROP COLUMN dup_of") + conn.execute("PRAGMA user_version = 17") + + migrated = Store(path) + + assert migrated.migrations_applied == [18] + assert dup_of(migrated) == {first.uid: None, second.uid: first.uid} + with sqlite3.connect(path) as conn: + assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'" + ).fetchone() is not None diff --git a/tests/test_pull.py b/tests/test_pull.py index bcf6eda..25bdfc5 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -31,11 +31,11 @@ class FakeConfig: searches: ClassVar[list[object]] = [] -def _item(board: str, slug: str, job_id: str) -> Item: +def _item(board: str, slug: str, job_id: str, company: str = "Acme") -> Item: item = Item( source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}", title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), - raw={"company": "Acme", "place": "Berlin", "department": "", + raw={"company": company, "place": "Berlin", "department": "", "function": "engineering", "remote": None, "salary_text": "", "board": board, "slug": slug}) item.source = "ats" @@ -126,7 +126,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( store.upsert_role(_item("greenhouse", "acme", "1"), sub) store.upsert_role(_item("greenhouse", "acme", "2"), sub) - store.upsert_role(_item("greenhouse", "other", "9"), other) + store.upsert_role(_item("greenhouse", "other", "9", company="Other"), other) # acme answered this run and now lists only "1"; other's board failed # (or its batch never got to it) and stamped nothing. @@ -140,7 +140,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0, - "roles": 1, "closed": 1} + "roles": 1, "closed": 1, "copies": 0} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} @@ -161,10 +161,39 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0, - "roles": 0, "closed": 0} + "roles": 0, "closed": 0, "copies": 0} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} +def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None: + """The settle's last step: a board's repeats of one job are one role, + and when the kept one closes the next copy is kept in its place.""" + sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", + company="Acme", board="greenhouse") + for job_id in ("1", "2", "3"): + store.upsert_role(_item("greenhouse", "acme", job_id), sub) + store.state["ats:ok:greenhouse:acme"] = "run-1" + store.state["ats:complete:greenhouse:acme"] = "1" + store.state["ats:ids:greenhouse:acme"] = json.dumps( + ["greenhouse:acme:1", "greenhouse:acme:2", "greenhouse:acme:3"]) + + class FakeTally: + results = ({"roles": 3, "run": "run-1"},) + + result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] + assert result["copies"] == 2 + [kept] = store.roles_page() + assert kept.copies == 2 + + gone = store.role(kept.uid).source_id # type: ignore[union-attr] + store.state["ats:ids:greenhouse:acme"] = json.dumps( + [f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone]) + result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] + assert (result["closed"], result["copies"]) == (1, 1) + [after] = store.roles_page() + assert after.uid != kept.uid and after.copies == 1 + + # ---------------------------------------------- whole pulls, fetch stubbed def _row(job_id: str) -> dict[str, object]: diff --git a/tests/test_storage.py b/tests/test_storage.py index 0015b04..9c08464 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 17 + assert LATEST == 18 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -88,8 +88,15 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]: return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")] +def _undo_v18(conn: sqlite3.Connection) -> None: + """A fresh file put back to the v17 shape: no copies.""" + conn.execute("DROP INDEX idx_roles_dup_of") + conn.execute("ALTER TABLE roles DROP COLUMN dup_of") + + def _undo_v17(conn: sqlite3.Connection) -> None: """A fresh file put back to the v16 shape: no folded place on a role.""" + _undo_v18(conn) conn.execute("ALTER TABLE roles DROP COLUMN place_fold") @@ -159,7 +166,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated( migrated = Store(path) - assert migrated.migrations_applied == [15, 16, 17] + assert migrated.migrations_applied == [15, 16, 17, 18] acme = migrated.company("acme") assert acme is not None assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example") @@ -364,7 +371,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14, 15, 16, 17] + assert migrated.migrations_applied == [14, 15, 16, 17, 18] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -388,7 +395,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None migrated = Store(path) - assert migrated.migrations_applied == [16, 17] + assert migrated.migrations_applied == [16, 17, 18] acme = migrated.company("acme") assert acme is not None assert (acme.careers_url, acme.notes, acme.hand_reason) == ( @@ -417,7 +424,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( migrated = Store(path) - assert migrated.migrations_applied == [17] + assert migrated.migrations_applied == [17, 18] with sqlite3.connect(path) as conn: rows = {uid: (fold, level) for uid, fold, level in conn.execute( "SELECT uid, place_fold, seniority FROM roles")} diff --git a/tests/test_web.py b/tests/test_web.py index bfa4e57..4d62051 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -1159,7 +1159,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns( "uid", "company_key", "company", "title", "place", "country", "remote", "function", "seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state", "original_url", "posted_at", "score", "gated_reason", "prompt", "vote", - "voted_at", "read_at", "tags", "excerpt", + "voted_at", "read_at", "tags", "excerpt", "copies", } assert row["uid"] == uid assert row["company"] == "Acme" and row["place"] == "Berlin, Germany" @@ -1644,3 +1644,29 @@ def test_summarise_queues_one_role_and_the_job_writes_it( assert asked == [[uid]] assert store.role(uid).summary == "Backend role in Berlin, Python." # type: ignore[union-attr] assert client.post(f"/api/roles/{uid}/summarise").status_code == 409 + + +def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role( + client: TestClient, store: Store) -> None: + """The contract the reader and the feed row build on: `copies` on a row + is a count, on `GET /api/roles/{uid}` the list of open copies, and a copy + still opens, with `dup_of`.""" + board = posting("board", company="Acme", place="Ballerup", board="greenhouse") + board.body = "The whole posting." + search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex") + for item in (board, search, posting("alone", company="Other")): + store.upsert_role(item) + store.rebuild_dupes() + + page = client.get("/api/roles").json() + assert page["total"] == 2 + assert {r["uid"]: r["copies"] for r in page["items"]} == { + board.uid: 1, posting("alone").uid: 0} + + kept = client.get(f"/api/roles/{board.uid}").json() + assert kept["dup_of"] is None + assert kept["copies"] == [{ + "uid": search.uid, "host": "jobindex", "place": "Ballerup, Denmark", + "original_url": search.location, "posted_at": kept["copies"][0]["posted_at"]}] + copy = client.get(f"/api/roles/{search.uid}").json() + assert copy["dup_of"] == board.uid and copy["copies"] == []