Merge branch 'wp-e2-copies'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
695d38c339
6 files changed
+336
-16
No files matched your search
+75
-7
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -287,6 +287,13 @@ def _int_or_none(value: Any) -> int | None:
|
||||
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
|
||||
|
||||
|
||||
#: A copy that is not one place of its kept role: the copies a role lists
|
||||
#: under "Also posted" and counts as "x N". `{kept}` is the kept role's uid
|
||||
#: as SQL, and a place copy is the one whose link the kept role holds.
|
||||
_NOT_A_PLACE = ("original_url NOT IN (SELECT url FROM role_places "
|
||||
"WHERE uid = {kept} AND url != '')")
|
||||
|
||||
|
||||
class DupeCounts(NamedTuple):
|
||||
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
|
||||
open roles that are a copy, and rows whose `dup_of` it rewrote."""
|
||||
@@ -296,8 +303,35 @@ class DupeCounts(NamedTuple):
|
||||
changed: int
|
||||
|
||||
|
||||
def _assign_only(rows: Iterable[dupes.Copy]) -> dupes.Folded:
|
||||
return dupes.Folded(dupes.assign(rows), frozenset())
|
||||
|
||||
|
||||
def _sync_copy_places(conn: sqlite3.Connection, placed: dict[str, str | None]) -> None:
|
||||
"""Make the `role_places` rows with a `url` those of the copies `placed`
|
||||
(`copy uid -> kept uid`): each own place of a copy, on the kept role,
|
||||
with the copy's link. Written only when they differ from what is stored,
|
||||
so an unchanged database is not touched. A copy that closed, or a group
|
||||
that dissolved, is no longer in `placed`, and its rows go."""
|
||||
want: set[tuple[str, int, str, str, int | None, str]] = set()
|
||||
if placed:
|
||||
marks = ",".join("?" * len(placed))
|
||||
for uid, pos, text, fold, place_id, url in conn.execute(
|
||||
f"""SELECT rp.uid, rp.position, rp.text, rp.fold, rp.place_id, r.original_url
|
||||
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
||||
WHERE rp.url = '' AND rp.uid IN ({marks})""", list(placed)):
|
||||
want.add((str(placed[uid]), pos, text, fold, place_id, url))
|
||||
have = set(conn.execute(
|
||||
"SELECT uid, position, text, fold, place_id, url FROM role_places WHERE url != ''"))
|
||||
if want != have:
|
||||
conn.execute("DELETE FROM role_places WHERE url != ''")
|
||||
conn.executemany(
|
||||
"INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)", sorted(want, key=lambda r: (r[0], r[5], r[1], r[2])))
|
||||
|
||||
|
||||
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
||||
"""Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside
|
||||
"""Recompute `roles.dup_of` from scratch with `app.dupes.fold`, inside
|
||||
the caller's transaction: `Store.rebuild_dupes` and migration 18.
|
||||
|
||||
Only the open roles of a `(company_key, title_key)` with more than one
|
||||
@@ -315,26 +349,40 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
||||
them moves to it unless that agent already voted on it; a vote that
|
||||
cannot move stays where it is. An unchanged group is not touched, so
|
||||
marking a kept role unread again sticks. Applications keep their
|
||||
`role_uid`, and scores of a copy go at the agent's next `score`."""
|
||||
`role_uid`, and scores of a copy go at the agent's next `score`.
|
||||
|
||||
A copy the host published for another place (`dupes.Folded.placed`)
|
||||
leaves its places on the kept role as `role_places` rows with the copy's
|
||||
link, see `_sync_copy_places`."""
|
||||
# Migration 18 runs this on a file that has no `teaser` and no
|
||||
# `role_places` yet: there it groups as before and folds no places.
|
||||
current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)"))
|
||||
text = "COALESCE(NULLIF(description, ''), teaser)" if current else "description"
|
||||
rows = conn.execute(
|
||||
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
||||
description != '' AS described, posted_at
|
||||
description != '' AS described, posted_at,
|
||||
SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url
|
||||
FROM roles
|
||||
WHERE closed_at IS NULL AND company_key != ''
|
||||
AND (company_key, title_key) IN (
|
||||
SELECT company_key, title_key FROM roles
|
||||
WHERE closed_at IS NULL AND company_key != ''
|
||||
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
|
||||
new = dupes.assign(
|
||||
made = dupes.fold if current else _assign_only
|
||||
folded = made(
|
||||
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
||||
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None)
|
||||
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None,
|
||||
text=r[8] or "", url=r[9] or "")
|
||||
for r in rows)
|
||||
new = folded.dup_of
|
||||
old: dict[str, str | None] = dict.fromkeys(new)
|
||||
old.update((r[0], r[1]) for r in conn.execute(
|
||||
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
|
||||
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
|
||||
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
|
||||
|
||||
if current:
|
||||
_sync_copy_places(conn, {uid: new[uid] for uid in folded.placed})
|
||||
changed = {uid for _, uid in updates}
|
||||
members: dict[str, set[str]] = {}
|
||||
for uid, dup in new.items():
|
||||
@@ -713,7 +761,8 @@ class Store:
|
||||
or a role without any. See `Role.copies`."""
|
||||
return [RoleCopy.from_row(r) for r in self._rows(
|
||||
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
|
||||
WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))]
|
||||
WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")}
|
||||
ORDER BY posted_at, uid""", (uid, uid))]
|
||||
|
||||
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
|
||||
"""`uid -> open copies` for a page of kept roles, one grouped query
|
||||
@@ -726,8 +775,25 @@ class Store:
|
||||
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
|
||||
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
|
||||
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
|
||||
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
|
||||
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
||||
|
||||
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
|
||||
"""`uid -> the line the feed shows` for a page of roles that have
|
||||
more than one place (`dupes.place_line`); a role with one place or
|
||||
none is absent and keeps `roles.place`."""
|
||||
if not uids:
|
||||
return {}
|
||||
found: dict[str, list[str]] = {}
|
||||
for r in self._rows(
|
||||
"""SELECT uid, text FROM role_places
|
||||
WHERE uid IN (SELECT value FROM json_each(?))
|
||||
ORDER BY uid, url != '', url, position, rowid""",
|
||||
(json.dumps(list(uids)),)):
|
||||
found.setdefault(str(r["uid"]), []).append(str(r["text"]))
|
||||
return {uid: dupes.place_line(texts) for uid, texts in found.items()
|
||||
if len(set(texts)) > 1}
|
||||
|
||||
def rebuild_dupes(self) -> DupeCounts:
|
||||
"""Recompute which open roles are copies of one posting, in one
|
||||
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
|
||||
@@ -972,9 +1038,11 @@ class Store:
|
||||
found.append(RoleRow.from_row(row))
|
||||
by_uid = self._tags_by_uid([f.uid for f in found])
|
||||
copies = self._copy_counts([f.uid for f in found])
|
||||
lines = self._place_lines([f.uid for f in found])
|
||||
for f in found:
|
||||
f.tags = by_uid.get(f.uid, [])
|
||||
f.copies = copies.get(f.uid, 0)
|
||||
f.place = lines.get(f.uid, f.place)
|
||||
return found
|
||||
|
||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||
|
||||
Reference in new issue
Block a user