Store copies of one posting as roles.dup_of and leave them out of the feed, the counts and the model stages
Migration 18 adds roles.dup_of with a partial index and runs the first rebuild. Store.rebuild_dupes feeds app.dupes.assign the open roles of every (company_key, title_key) with more than one, rewrites dup_of where it changed, clears it on closed roles, and hands read_at and votes to a changed group's kept role. The pull's settle and `app dupes` run it. RoleRow.copies counts a row's open copies; Role.copies lists them and Role.dup_of names a copy's kept role. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
55e353e764
commit
545c932f7b
10 files changed
+490
-41
No files matched your search
+160
-26
@@ -14,16 +14,17 @@ from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from typing import Any, Literal, NamedTuple
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
|
||||
from desk.models import Item
|
||||
from app import places
|
||||
from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
|
||||
ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
@@ -50,8 +51,9 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
|
||||
#: never answers is otherwise paid for on every pull for as long as it exists.
|
||||
GIVE_UP_AFTER = 3
|
||||
|
||||
#: A role `extract` has yet to read: never answered, still open, not given up.
|
||||
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
|
||||
#: A role `extract` has yet to read: never answered, still open, no copy of
|
||||
#: another (`rebuild_dupes`), not given up.
|
||||
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL AND dup_of IS NULL "
|
||||
f"AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
||||
@@ -226,6 +228,93 @@ def _int_or_none(value: Any) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
#: The host `app.dupes.Copy.host` and `RoleCopy.host` read: the provider of
|
||||
#: a search's row, else the ATS a board runs on, as `posted_host` reads it.
|
||||
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
|
||||
|
||||
|
||||
class DupeCounts(NamedTuple):
|
||||
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
|
||||
open roles that are a copy, and rows whose `dup_of` it rewrote."""
|
||||
|
||||
groups: int
|
||||
copies: int
|
||||
changed: int
|
||||
|
||||
|
||||
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
||||
"""Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside
|
||||
the caller's transaction: `Store.rebuild_dupes` and migration 18.
|
||||
|
||||
Only the open roles of a `(company_key, title_key)` with more than one
|
||||
open role can be a copy, a few thousand of the ~34k, so only those are
|
||||
read. A row whose `dup_of` differs from the rule's answer is rewritten,
|
||||
and a `dup_of` left on a closed role, or on one no longer in such a pair,
|
||||
is cleared: a copy never outlives its posting, and a closed kept role
|
||||
hands its group to the next copy here.
|
||||
|
||||
Where a group changed (a member's `dup_of` was rewritten), its kept role
|
||||
takes over what was done to the posting under another uid. The sources
|
||||
are the group's members and the roles they were copies of before, so a
|
||||
closed kept role still hands over. The kept role takes the earliest
|
||||
`read_at` of them when it has none, and each agent's latest vote among
|
||||
them moves to it unless that agent already voted on it; a vote that
|
||||
cannot move stays where it is. An unchanged group is not touched, so
|
||||
marking a kept role unread again sticks. Applications keep their
|
||||
`role_uid`, and scores of a copy go at the agent's next `score`."""
|
||||
rows = conn.execute(
|
||||
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
||||
description != '' AS described, posted_at
|
||||
FROM roles
|
||||
WHERE closed_at IS NULL AND company_key != ''
|
||||
AND (company_key, title_key) IN (
|
||||
SELECT company_key, title_key FROM roles
|
||||
WHERE closed_at IS NULL AND company_key != ''
|
||||
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
|
||||
new = dupes.assign(
|
||||
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
||||
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None)
|
||||
for r in rows)
|
||||
old: dict[str, str | None] = dict.fromkeys(new)
|
||||
old.update((r[0], r[1]) for r in conn.execute(
|
||||
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
|
||||
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
|
||||
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
|
||||
|
||||
changed = {uid for _, uid in updates}
|
||||
members: dict[str, set[str]] = {}
|
||||
for uid, dup in new.items():
|
||||
members.setdefault(dup or uid, set()).add(uid)
|
||||
groups = {kept: group for kept, group in members.items() if len(group) > 1}
|
||||
for kept, group in groups.items():
|
||||
if not group & changed:
|
||||
continue
|
||||
sources = (group | {old.get(uid) or uid for uid in group}) - {kept}
|
||||
_carry(conn, kept, sources)
|
||||
return DupeCounts(groups=len(groups), copies=sum(dup is not None for dup in new.values()),
|
||||
changed=len(updates))
|
||||
|
||||
|
||||
def _carry(conn: sqlite3.Connection, kept: str, sources: set[str]) -> None:
|
||||
"""`rebuild_dupes`' hand-over to a group's kept role: the earliest
|
||||
`read_at` of `sources` if it has none, and each agent's latest vote among
|
||||
them unless it has one of its own."""
|
||||
wanted = json.dumps(sorted(sources))
|
||||
conn.execute(
|
||||
"""UPDATE roles SET read_at = (SELECT MIN(read_at) FROM roles
|
||||
WHERE uid IN (SELECT value FROM json_each(?)))
|
||||
WHERE uid = ? AND read_at IS NULL""", (wanted, kept))
|
||||
voted = {r[0] for r in conn.execute("SELECT agent_id FROM role_votes WHERE uid = ?", (kept,))}
|
||||
moves: dict[int, str] = {}
|
||||
for agent_id, uid in conn.execute(
|
||||
"""SELECT agent_id, uid FROM role_votes WHERE uid IN (SELECT value FROM json_each(?))
|
||||
ORDER BY voted_at, uid""", (wanted,)):
|
||||
if agent_id not in voted:
|
||||
moves[agent_id] = uid # the latest vote wins, by the order above
|
||||
conn.executemany("UPDATE role_votes SET uid = ? WHERE agent_id = ? AND uid = ?",
|
||||
[(kept, agent_id, uid) for agent_id, uid in moves.items()])
|
||||
|
||||
|
||||
class _StateMap(MutableMapping[str, str]):
|
||||
"""`Store.state`: the key/value table, read and written one key at a time."""
|
||||
|
||||
@@ -443,8 +532,39 @@ class Store:
|
||||
return None
|
||||
role = ScoredRole.from_row(row)
|
||||
role.tags = self._tags_of(uid)
|
||||
role.copies = self._copies_of(uid)
|
||||
return role
|
||||
|
||||
def _copies_of(self, uid: str) -> list[RoleCopy]:
|
||||
"""The open copies of one kept role, oldest first; empty for a copy
|
||||
or a role without any. See `Role.copies`."""
|
||||
return [RoleCopy.from_row(r) for r in self._rows(
|
||||
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
|
||||
WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))]
|
||||
|
||||
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
|
||||
"""`uid -> open copies` for a page of kept roles, one grouped query
|
||||
over `idx_roles_dup_of`. A role without copies is absent. The index
|
||||
is named because the planner otherwise takes `idx_roles_closed` for
|
||||
`closed_at IS NULL` and reads every open role, 19 ms against 0.1 on
|
||||
the live database."""
|
||||
if not uids:
|
||||
return {}
|
||||
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
|
||||
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
|
||||
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
|
||||
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
||||
|
||||
def rebuild_dupes(self) -> DupeCounts:
|
||||
"""Recompute which open roles are copies of one posting, in one
|
||||
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
|
||||
because it reads before it writes: a deferred transaction that a
|
||||
pull's upsert wrote under in between could not take the write lock
|
||||
and would fail rather than wait."""
|
||||
with self.transaction() as conn:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
return rebuild_dupes(conn)
|
||||
|
||||
def set_read(self, uid: str, read: bool = True) -> Role:
|
||||
"""Mark a role read, or unread again. Stamps `read_at` only the first
|
||||
time — a role already read keeps the moment it was first opened —
|
||||
@@ -677,9 +797,11 @@ class Store:
|
||||
row = dict(r)
|
||||
row["excerpt"] = excerpt(row["excerpt"])
|
||||
found.append(RoleRow.from_row(row))
|
||||
by_uid = self._tags_by_uid([r.uid for r in found])
|
||||
for r in found:
|
||||
r.tags = by_uid.get(r.uid, [])
|
||||
by_uid = self._tags_by_uid([f.uid for f in found])
|
||||
copies = self._copy_counts([f.uid for f in found])
|
||||
for f in found:
|
||||
f.tags = by_uid.get(f.uid, [])
|
||||
f.copies = copies.get(f.uid, 0)
|
||||
return found
|
||||
|
||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||
@@ -749,8 +871,10 @@ class Store:
|
||||
pay_unknown: bool = False) -> tuple[str, list[Any]]:
|
||||
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
||||
its parameters in order. The agent's votes are always joined as `v`
|
||||
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
|
||||
where: list[str] = []
|
||||
(see `_WITH_VOTES`); its scores as `s` only when ordering by them. A
|
||||
copy of another role (`rebuild_dupes`) is never listed: its kept role
|
||||
stands for it and counts its copies."""
|
||||
where: list[str] = ["roles.dup_of IS NULL"]
|
||||
params: list[Any] = []
|
||||
if company:
|
||||
where.append("company LIKE ? ESCAPE '\\'")
|
||||
@@ -831,7 +955,7 @@ class Store:
|
||||
first and then alphabetically. What `/api/filters` serves; the
|
||||
companies `agent_id` hid are left out unless `show_muted`, the same
|
||||
default as `roles`, and with no agent nothing is hidden."""
|
||||
where = "r.closed_at IS NULL"
|
||||
where = "r.closed_at IS NULL AND r.dup_of IS NULL"
|
||||
params: list[Any] = []
|
||||
if not show_muted and agent_id is not None:
|
||||
where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
|
||||
@@ -913,6 +1037,7 @@ class Store:
|
||||
WHERE p.agent_id = a.id
|
||||
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
AND top.dup_of IS NULL
|
||||
WHERE s.gated_reason = '')
|
||||
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
|
||||
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
|
||||
@@ -924,7 +1049,8 @@ class Store:
|
||||
batch asks again when it runs and must finish what it was dealt;
|
||||
otherwise only the roles in `_top_scope`."""
|
||||
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")]
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
|
||||
.replace("dup_of", "r.dup_of")]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
@@ -1033,6 +1159,7 @@ class Store:
|
||||
top, params = self._top_scope("summarise")
|
||||
where += [
|
||||
"r.closed_at IS NULL",
|
||||
"r.dup_of IS NULL",
|
||||
f"""NOT EXISTS (SELECT 1 FROM state
|
||||
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
|
||||
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
|
||||
@@ -1106,11 +1233,12 @@ class Store:
|
||||
|
||||
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
||||
"""Open roles with no vector from `model`, newest first, so a backlog
|
||||
embedded from the top improves what is on screen first."""
|
||||
embedded from the top improves what is on screen first. A copy is
|
||||
not embedded: it is never scored."""
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
"""SELECT r.* FROM roles r
|
||||
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
|
||||
WHERE v.uid IS NULL AND r.closed_at IS NULL
|
||||
WHERE v.uid IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL
|
||||
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
|
||||
|
||||
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
|
||||
@@ -1239,10 +1367,11 @@ class Store:
|
||||
# --------------------------------------------------------------- scores
|
||||
|
||||
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
|
||||
"""Every open role of a company this agent has not hidden, with the
|
||||
agent's affinity for the company and how the role meets each gate: 1
|
||||
match, 0 no match, -1 the column is empty. The gates are evaluated
|
||||
here, in SQL, so a `require` gate is the `WHERE` it reads as, written
|
||||
"""Every open role of a company this agent has not hidden, copies
|
||||
(`rebuild_dupes`) aside, with the agent's affinity for the company
|
||||
and how the role meets each gate: 1 match, 0 no match, -1 the column
|
||||
is empty. The gates are evaluated here, in SQL, so a `require` gate is
|
||||
the `WHERE` it reads as, written
|
||||
as a column so a gated role keeps its row and reason."""
|
||||
columns: list[str] = []
|
||||
params: list[Any] = []
|
||||
@@ -1257,7 +1386,8 @@ class Store:
|
||||
f"""SELECT {select} FROM roles r
|
||||
LEFT JOIN agent_companies ac
|
||||
ON ac.company_key = r.company_key AND ac.agent_id = ?
|
||||
WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""",
|
||||
WHERE r.closed_at IS NULL AND r.dup_of IS NULL
|
||||
AND COALESCE(ac.affinity, 0) <> -1""",
|
||||
[*params, agent_id])
|
||||
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
|
||||
affinity=int(r["affinity"]),
|
||||
@@ -1403,10 +1533,11 @@ class Store:
|
||||
|
||||
def open_role_counts(self) -> dict[str, int]:
|
||||
"""`company_key -> open role count`, for the company list's row
|
||||
count. A key with no open roles is simply absent."""
|
||||
count. A key with no open roles is simply absent. Copies are not
|
||||
counted, so the number is the rows the feed lists for the company."""
|
||||
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
|
||||
"SELECT company_key, COUNT(*) AS n FROM roles "
|
||||
"WHERE closed_at IS NULL GROUP BY company_key")}
|
||||
"WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")}
|
||||
|
||||
def open_companies(self, agent_id: int | None = None
|
||||
) -> list[tuple[str, list[str], int, int]]:
|
||||
@@ -1415,7 +1546,7 @@ class Store:
|
||||
company row's name and then every spelling its roles carry, commonest
|
||||
first, so a search finds a company by any of them. A company
|
||||
`agent_id` hid is left out; `affinity` is that agent's, 0 without
|
||||
one."""
|
||||
one. Copies are not counted, as in the feed."""
|
||||
rows = self._rows(
|
||||
"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n,
|
||||
c.name AS own, COALESCE(ac.affinity, 0) AS affinity
|
||||
@@ -1423,7 +1554,7 @@ class Store:
|
||||
LEFT JOIN companies c ON c.key = r.company_key
|
||||
LEFT JOIN agent_companies ac ON ac.company_key = r.company_key
|
||||
AND ac.agent_id = ?
|
||||
WHERE r.closed_at IS NULL AND r.company_key != ''
|
||||
WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != ''
|
||||
AND COALESCE(ac.affinity, 0) != -1
|
||||
GROUP BY r.company_key, r.company
|
||||
ORDER BY r.company_key, n DESC, r.company""", (agent_id,))
|
||||
@@ -1439,13 +1570,16 @@ class Store:
|
||||
|
||||
def open_places(self) -> list[tuple[str, str, int]]:
|
||||
"""`(place, country, open roles)` for every place line an open role
|
||||
carries, what `places.suggest` reads."""
|
||||
carries, what `places.suggest` reads. Copies are not counted, as in
|
||||
the feed."""
|
||||
return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows(
|
||||
"SELECT place, country, COUNT(*) AS n FROM roles "
|
||||
"WHERE closed_at IS NULL AND place != '' GROUP BY place, country")]
|
||||
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
|
||||
"GROUP BY place, country")]
|
||||
|
||||
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
||||
"""Per subscription id, its open roles counted by `country`."""
|
||||
"""Per subscription id, its open roles counted by `country`, copies
|
||||
included: what a board or a search holds, not what the feed lists."""
|
||||
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
|
||||
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
||||
GROUP BY subscription_id, country""")
|
||||
|
||||
Reference in new issue
Block a user