Merge branch 'wp-e2-copies'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:49:10 +02:00
commit 695d38c339
6 files changed
+336 -16

No files matched your search

+75 -7
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import json
import sqlite3
from collections.abc import Callable, Iterator, MutableMapping, Sequence
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
from contextlib import closing, contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -287,6 +287,13 @@ def _int_or_none(value: Any) -> int | None:
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
#: A copy that is not one place of its kept role: the copies a role lists
#: under "Also posted" and counts as "x N". `{kept}` is the kept role's uid
#: as SQL, and a place copy is the one whose link the kept role holds.
_NOT_A_PLACE = ("original_url NOT IN (SELECT url FROM role_places "
"WHERE uid = {kept} AND url != '')")
class DupeCounts(NamedTuple):
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
open roles that are a copy, and rows whose `dup_of` it rewrote."""
@@ -296,8 +303,35 @@ class DupeCounts(NamedTuple):
changed: int
def _assign_only(rows: Iterable[dupes.Copy]) -> dupes.Folded:
return dupes.Folded(dupes.assign(rows), frozenset())
def _sync_copy_places(conn: sqlite3.Connection, placed: dict[str, str | None]) -> None:
"""Make the `role_places` rows with a `url` those of the copies `placed`
(`copy uid -> kept uid`): each own place of a copy, on the kept role,
with the copy's link. Written only when they differ from what is stored,
so an unchanged database is not touched. A copy that closed, or a group
that dissolved, is no longer in `placed`, and its rows go."""
want: set[tuple[str, int, str, str, int | None, str]] = set()
if placed:
marks = ",".join("?" * len(placed))
for uid, pos, text, fold, place_id, url in conn.execute(
f"""SELECT rp.uid, rp.position, rp.text, rp.fold, rp.place_id, r.original_url
FROM role_places rp JOIN roles r ON r.uid = rp.uid
WHERE rp.url = '' AND rp.uid IN ({marks})""", list(placed)):
want.add((str(placed[uid]), pos, text, fold, place_id, url))
have = set(conn.execute(
"SELECT uid, position, text, fold, place_id, url FROM role_places WHERE url != ''"))
if want != have:
conn.execute("DELETE FROM role_places WHERE url != ''")
conn.executemany(
"INSERT INTO role_places (uid, position, text, fold, place_id, url) "
"VALUES (?, ?, ?, ?, ?, ?)", sorted(want, key=lambda r: (r[0], r[5], r[1], r[2])))
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
"""Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside
"""Recompute `roles.dup_of` from scratch with `app.dupes.fold`, inside
the caller's transaction: `Store.rebuild_dupes` and migration 18.
Only the open roles of a `(company_key, title_key)` with more than one
@@ -315,26 +349,40 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
them moves to it unless that agent already voted on it; a vote that
cannot move stays where it is. An unchanged group is not touched, so
marking a kept role unread again sticks. Applications keep their
`role_uid`, and scores of a copy go at the agent's next `score`."""
`role_uid`, and scores of a copy go at the agent's next `score`.
A copy the host published for another place (`dupes.Folded.placed`)
leaves its places on the kept role as `role_places` rows with the copy's
link, see `_sync_copy_places`."""
# Migration 18 runs this on a file that has no `teaser` and no
# `role_places` yet: there it groups as before and folds no places.
current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)"))
text = "COALESCE(NULLIF(description, ''), teaser)" if current else "description"
rows = conn.execute(
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
description != '' AS described, posted_at
description != '' AS described, posted_at,
SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url
FROM roles
WHERE closed_at IS NULL AND company_key != ''
AND (company_key, title_key) IN (
SELECT company_key, title_key FROM roles
WHERE closed_at IS NULL AND company_key != ''
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
new = dupes.assign(
made = dupes.fold if current else _assign_only
folded = made(
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None)
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None,
text=r[8] or "", url=r[9] or "")
for r in rows)
new = folded.dup_of
old: dict[str, str | None] = dict.fromkeys(new)
old.update((r[0], r[1]) for r in conn.execute(
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
if current:
_sync_copy_places(conn, {uid: new[uid] for uid in folded.placed})
changed = {uid for _, uid in updates}
members: dict[str, set[str]] = {}
for uid, dup in new.items():
@@ -713,7 +761,8 @@ class Store:
or a role without any. See `Role.copies`."""
return [RoleCopy.from_row(r) for r in self._rows(
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))]
WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")}
ORDER BY posted_at, uid""", (uid, uid))]
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
"""`uid -> open copies` for a page of kept roles, one grouped query
@@ -726,8 +775,25 @@ class Store:
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
"GROUP BY dup_of", (json.dumps(list(uids)),))}
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
"""`uid -> the line the feed shows` for a page of roles that have
more than one place (`dupes.place_line`); a role with one place or
none is absent and keeps `roles.place`."""
if not uids:
return {}
found: dict[str, list[str]] = {}
for r in self._rows(
"""SELECT uid, text FROM role_places
WHERE uid IN (SELECT value FROM json_each(?))
ORDER BY uid, url != '', url, position, rowid""",
(json.dumps(list(uids)),)):
found.setdefault(str(r["uid"]), []).append(str(r["text"]))
return {uid: dupes.place_line(texts) for uid, texts in found.items()
if len(set(texts)) > 1}
def rebuild_dupes(self) -> DupeCounts:
"""Recompute which open roles are copies of one posting, in one
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
@@ -972,9 +1038,11 @@ class Store:
found.append(RoleRow.from_row(row))
by_uid = self._tags_by_uid([f.uid for f in found])
copies = self._copy_counts([f.uid for f in found])
lines = self._place_lines([f.uid for f in found])
for f in found:
f.tags = by_uid.get(f.uid, [])
f.copies = copies.get(f.uid, 0)
f.place = lines.get(f.uid, f.place)
return found
def count_roles(self, company: str | None = None, place: str | None = None,