Merge branch 'wp-e2-copies'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
695d38c339
6 files changed
+336
-16
No files matched your search
+97
-1
@@ -17,14 +17,30 @@ rows that each match all the others, never a chain: a row with no place
|
||||
matches Belgium and the Netherlands, and must not make them one role.
|
||||
|
||||
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
|
||||
it. Pure by design: no store, no SQL, no network, so the rule is tested on
|
||||
it.
|
||||
|
||||
A second rule folds what the first leaves apart on purpose: one posting per
|
||||
place. A host that lists one job once per office, each with its own id, gives
|
||||
rows with the same company and title whose towns never meet. `fold` merges
|
||||
those rows of one host into the kept role of the first group, when the texts
|
||||
do not say they are different jobs (`similar`), and reports them in
|
||||
`Folded.placed`: the store gives the kept role each such copy's place with the
|
||||
copy's own link. This holds for every host, not only readers that declare
|
||||
`READS.places == "copies"`: on the live database 1,739 company-and-title
|
||||
groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and
|
||||
Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts
|
||||
share fewer than `SAME_TEXT` of their words, or without a link to keep, or
|
||||
listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on
|
||||
the strings boards really write and the storage layer only feeds it rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import NamedTuple
|
||||
|
||||
from app import places
|
||||
from app.models import country_of
|
||||
@@ -48,6 +64,11 @@ class Copy:
|
||||
host: str
|
||||
has_description: bool
|
||||
posted_at: str | None
|
||||
#: The start of the description, else the teaser, for `similar`; '' when
|
||||
#: the row has neither.
|
||||
text: str = ""
|
||||
#: The link that opens this row on its host.
|
||||
url: str = ""
|
||||
|
||||
|
||||
def _country(c: Copy) -> str:
|
||||
@@ -118,3 +139,78 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
|
||||
if row is not keeper:
|
||||
result[row.uid] = keeper.uid
|
||||
return result
|
||||
|
||||
|
||||
#: The share of words two texts must have in common (Jaccard, over the first
|
||||
#: 3,000 characters) for one job in two places. Measured on the live
|
||||
#: database: copies of a posting share nearly all of it, two jobs that
|
||||
#: happen to share a title share little more than boilerplate, and a
|
||||
#: translation shares none, which keeps it apart.
|
||||
SAME_TEXT = 0.3
|
||||
|
||||
TEXT_CHARS = 3000
|
||||
|
||||
|
||||
@lru_cache(maxsize=8192)
|
||||
def _words(text: str) -> frozenset[str]:
|
||||
return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower()))
|
||||
|
||||
|
||||
def similar(a: str, b: str) -> bool:
|
||||
"""Whether two texts can be one job: a text that is missing says nothing
|
||||
against it, else they share at least `SAME_TEXT` of their words."""
|
||||
wa, wb = _words(a), _words(b)
|
||||
if not wa or not wb:
|
||||
return True
|
||||
return len(wa & wb) / len(wa | wb) >= SAME_TEXT
|
||||
|
||||
|
||||
class Folded(NamedTuple):
|
||||
"""`assign`'s answer plus the rows that are one place of another role:
|
||||
`placed` holds the uid of each copy that `fold` merged by place."""
|
||||
|
||||
dup_of: dict[str, str | None]
|
||||
placed: frozenset[str]
|
||||
|
||||
|
||||
def fold(rows: Iterable[Copy]) -> Folded:
|
||||
"""`assign`, then the per-place copies of each host folded in.
|
||||
|
||||
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
||||
again, in `keep_order`: a row joins the first group whose members are all
|
||||
of its host and `similar` to it. Only a row with a link can be folded, as
|
||||
its link is what the kept role keeps. Every row of a merged group points
|
||||
at its first row, the rows that pointed at a folded one included."""
|
||||
rows = list(rows)
|
||||
result = assign(rows)
|
||||
by_uid = {r.uid: r for r in rows}
|
||||
parts: dict[tuple[str, str], list[Copy]] = {}
|
||||
for uid, dup in result.items():
|
||||
row = by_uid[uid]
|
||||
if dup is None and row.company_key and row.url:
|
||||
parts.setdefault((row.company_key, row.title_key), []).append(row)
|
||||
moved: dict[str, str] = {}
|
||||
for part in parts.values():
|
||||
groups: list[list[Copy]] = []
|
||||
for row in sorted(part, key=keep_order):
|
||||
for group in groups:
|
||||
if all(m.host == row.host and similar(m.text, row.text) for m in group):
|
||||
group.append(row)
|
||||
break
|
||||
else:
|
||||
groups.append([row])
|
||||
for keeper, *others in groups:
|
||||
moved.update((other.uid, keeper.uid) for other in others)
|
||||
for uid, dup in result.items():
|
||||
if uid in moved:
|
||||
result[uid] = moved[uid]
|
||||
elif dup in moved:
|
||||
result[uid] = moved[dup]
|
||||
return Folded(result, frozenset(moved))
|
||||
|
||||
|
||||
def place_line(texts: Iterable[str]) -> str:
|
||||
"""The line a feed row shows for a role's places: the first, and "+N" for
|
||||
the N others. Repeated texts count once."""
|
||||
found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip()))
|
||||
return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found)
|
||||
@@ -942,6 +942,7 @@ body[data-reader="open"] .company-strip { display: none; }
|
||||
line-height: 1.4;
|
||||
}
|
||||
.also a b { display: block; color: var(--faint); font-size: .78rem; font-weight: 600; }
|
||||
.also span { display: block; margin-bottom: .55rem; color: var(--dim); font-size: .92rem; line-height: 1.4; }
|
||||
@media (hover: hover) {
|
||||
.also a:hover { color: var(--fg); }
|
||||
}
|
||||
|
||||
+42
-2
@@ -885,6 +885,42 @@
|
||||
</div>`;
|
||||
}
|
||||
|
||||
/* The role's places in the host's order, one entry for each place text a
|
||||
link opens: the copies' own links, and the role's link for its own. */
|
||||
function placeEntries(r) {
|
||||
const seen = new Set();
|
||||
const own = safeUrl(r.original_url);
|
||||
const out = [];
|
||||
for (const p of Array.isArray(r.places) ? r.places : []) {
|
||||
const url = p.url ? safeUrl(p.url) : own;
|
||||
const key = `${p.url || ""}\u0001${p.position}\u0001${p.text}`;
|
||||
if (!p.text || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push({ text: p.text, url });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/* The place line of the reader: the first place and "+N" for the others,
|
||||
as the feed's row shows it. */
|
||||
function placeLine(r) {
|
||||
const texts = [...new Set(placeEntries(r).map((e) => e.text))];
|
||||
return texts.length > 1 ? `${texts[0]} +${texts.length - 1}` : r.place || texts[0] || "";
|
||||
}
|
||||
|
||||
/* Every place of a role that has several, each a link to its own posting,
|
||||
laid out as "Also posted". */
|
||||
function placesList(r) {
|
||||
const entries = placeEntries(r);
|
||||
if (new Set(entries.map((e) => e.text)).size < 2) return "";
|
||||
return `<div class="also places">
|
||||
<h2 class="reader-label">Places</h2>
|
||||
${entries.map((e) => e.url
|
||||
? `<a href="${esc(e.url)}" target="_blank" rel="noopener">${esc(e.text)} ↗</a>`
|
||||
: `<span>${esc(e.text)}</span>`).join("")}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function drawRow(el, role) {
|
||||
const scored = role.score !== null && role.score !== undefined;
|
||||
const pct = scored ? Math.round(Math.min(1, Math.max(0, role.score)) * 100) : 0;
|
||||
@@ -1243,7 +1279,10 @@
|
||||
: !loaded
|
||||
? `<p class="reader-note">Loading…</p>`
|
||||
: text ? paragraphs(text)
|
||||
: `<p class="reader-note">The board gave no description for this one.</p>`;
|
||||
: (r.teaser || "").trim()
|
||||
? `<p class="reader-note reader-teaser">A short description from the board's
|
||||
list; the board gave no full text.</p>${paragraphs(r.teaser.trim())}`
|
||||
: `<p class="reader-note">The board gave no description for this one.</p>`;
|
||||
const pay = payDisplay(r);
|
||||
const careers = safeUrl(companies.get(r.company_key)?.careers_url);
|
||||
const waiting = state.summarising === state.readingUid;
|
||||
@@ -1264,7 +1303,7 @@
|
||||
${careers ? `<a class="reader-careers" href="${esc(careers)}" target="_blank" rel="noopener"
|
||||
title="The company's careers page">Careers ↗</a>` : ""}
|
||||
<span class="affinity-slot"></span>
|
||||
${r.place ? `<span>${esc(r.place)}</span>` : ""}
|
||||
${placeLine(r) ? `<span>${esc(placeLine(r))}</span>` : ""}
|
||||
${r.first_seen ? `<span title="First seen">${ago(r.first_seen)}</span>` : ""}
|
||||
${badges(r)}
|
||||
${tagChips(r.tags)}
|
||||
@@ -1287,6 +1326,7 @@
|
||||
<div class="prose">${body}</div>
|
||||
${out ? `<p class="reader-out"><a href="${esc(out)}" target="_blank" rel="noopener"
|
||||
>Read it on ${esc(host(out) || "the board")} ↗</a></p>` : ""}
|
||||
${loaded ? placesList(r) : ""}
|
||||
${loaded ? alsoPosted(r) : ""}
|
||||
</div>
|
||||
<div class="reader-foot">
|
||||
|
||||
+75
-7
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -287,6 +287,13 @@ def _int_or_none(value: Any) -> int | None:
|
||||
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
|
||||
|
||||
|
||||
#: A copy that is not one place of its kept role: the copies a role lists
|
||||
#: under "Also posted" and counts as "x N". `{kept}` is the kept role's uid
|
||||
#: as SQL, and a place copy is the one whose link the kept role holds.
|
||||
_NOT_A_PLACE = ("original_url NOT IN (SELECT url FROM role_places "
|
||||
"WHERE uid = {kept} AND url != '')")
|
||||
|
||||
|
||||
class DupeCounts(NamedTuple):
|
||||
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
|
||||
open roles that are a copy, and rows whose `dup_of` it rewrote."""
|
||||
@@ -296,8 +303,35 @@ class DupeCounts(NamedTuple):
|
||||
changed: int
|
||||
|
||||
|
||||
def _assign_only(rows: Iterable[dupes.Copy]) -> dupes.Folded:
|
||||
return dupes.Folded(dupes.assign(rows), frozenset())
|
||||
|
||||
|
||||
def _sync_copy_places(conn: sqlite3.Connection, placed: dict[str, str | None]) -> None:
|
||||
"""Make the `role_places` rows with a `url` those of the copies `placed`
|
||||
(`copy uid -> kept uid`): each own place of a copy, on the kept role,
|
||||
with the copy's link. Written only when they differ from what is stored,
|
||||
so an unchanged database is not touched. A copy that closed, or a group
|
||||
that dissolved, is no longer in `placed`, and its rows go."""
|
||||
want: set[tuple[str, int, str, str, int | None, str]] = set()
|
||||
if placed:
|
||||
marks = ",".join("?" * len(placed))
|
||||
for uid, pos, text, fold, place_id, url in conn.execute(
|
||||
f"""SELECT rp.uid, rp.position, rp.text, rp.fold, rp.place_id, r.original_url
|
||||
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
||||
WHERE rp.url = '' AND rp.uid IN ({marks})""", list(placed)):
|
||||
want.add((str(placed[uid]), pos, text, fold, place_id, url))
|
||||
have = set(conn.execute(
|
||||
"SELECT uid, position, text, fold, place_id, url FROM role_places WHERE url != ''"))
|
||||
if want != have:
|
||||
conn.execute("DELETE FROM role_places WHERE url != ''")
|
||||
conn.executemany(
|
||||
"INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)", sorted(want, key=lambda r: (r[0], r[5], r[1], r[2])))
|
||||
|
||||
|
||||
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
||||
"""Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside
|
||||
"""Recompute `roles.dup_of` from scratch with `app.dupes.fold`, inside
|
||||
the caller's transaction: `Store.rebuild_dupes` and migration 18.
|
||||
|
||||
Only the open roles of a `(company_key, title_key)` with more than one
|
||||
@@ -315,26 +349,40 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
||||
them moves to it unless that agent already voted on it; a vote that
|
||||
cannot move stays where it is. An unchanged group is not touched, so
|
||||
marking a kept role unread again sticks. Applications keep their
|
||||
`role_uid`, and scores of a copy go at the agent's next `score`."""
|
||||
`role_uid`, and scores of a copy go at the agent's next `score`.
|
||||
|
||||
A copy the host published for another place (`dupes.Folded.placed`)
|
||||
leaves its places on the kept role as `role_places` rows with the copy's
|
||||
link, see `_sync_copy_places`."""
|
||||
# Migration 18 runs this on a file that has no `teaser` and no
|
||||
# `role_places` yet: there it groups as before and folds no places.
|
||||
current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)"))
|
||||
text = "COALESCE(NULLIF(description, ''), teaser)" if current else "description"
|
||||
rows = conn.execute(
|
||||
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
||||
description != '' AS described, posted_at
|
||||
description != '' AS described, posted_at,
|
||||
SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url
|
||||
FROM roles
|
||||
WHERE closed_at IS NULL AND company_key != ''
|
||||
AND (company_key, title_key) IN (
|
||||
SELECT company_key, title_key FROM roles
|
||||
WHERE closed_at IS NULL AND company_key != ''
|
||||
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
|
||||
new = dupes.assign(
|
||||
made = dupes.fold if current else _assign_only
|
||||
folded = made(
|
||||
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
||||
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None)
|
||||
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None,
|
||||
text=r[8] or "", url=r[9] or "")
|
||||
for r in rows)
|
||||
new = folded.dup_of
|
||||
old: dict[str, str | None] = dict.fromkeys(new)
|
||||
old.update((r[0], r[1]) for r in conn.execute(
|
||||
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
|
||||
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
|
||||
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
|
||||
|
||||
if current:
|
||||
_sync_copy_places(conn, {uid: new[uid] for uid in folded.placed})
|
||||
changed = {uid for _, uid in updates}
|
||||
members: dict[str, set[str]] = {}
|
||||
for uid, dup in new.items():
|
||||
@@ -713,7 +761,8 @@ class Store:
|
||||
or a role without any. See `Role.copies`."""
|
||||
return [RoleCopy.from_row(r) for r in self._rows(
|
||||
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
|
||||
WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))]
|
||||
WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")}
|
||||
ORDER BY posted_at, uid""", (uid, uid))]
|
||||
|
||||
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
|
||||
"""`uid -> open copies` for a page of kept roles, one grouped query
|
||||
@@ -726,8 +775,25 @@ class Store:
|
||||
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
|
||||
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
|
||||
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
|
||||
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
|
||||
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
||||
|
||||
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
|
||||
"""`uid -> the line the feed shows` for a page of roles that have
|
||||
more than one place (`dupes.place_line`); a role with one place or
|
||||
none is absent and keeps `roles.place`."""
|
||||
if not uids:
|
||||
return {}
|
||||
found: dict[str, list[str]] = {}
|
||||
for r in self._rows(
|
||||
"""SELECT uid, text FROM role_places
|
||||
WHERE uid IN (SELECT value FROM json_each(?))
|
||||
ORDER BY uid, url != '', url, position, rowid""",
|
||||
(json.dumps(list(uids)),)):
|
||||
found.setdefault(str(r["uid"]), []).append(str(r["text"]))
|
||||
return {uid: dupes.place_line(texts) for uid, texts in found.items()
|
||||
if len(set(texts)) > 1}
|
||||
|
||||
def rebuild_dupes(self) -> DupeCounts:
|
||||
"""Recompute which open roles are copies of one posting, in one
|
||||
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
|
||||
@@ -972,9 +1038,11 @@ class Store:
|
||||
found.append(RoleRow.from_row(row))
|
||||
by_uid = self._tags_by_uid([f.uid for f in found])
|
||||
copies = self._copy_counts([f.uid for f in found])
|
||||
lines = self._place_lines([f.uid for f in found])
|
||||
for f in found:
|
||||
f.tags = by_uid.get(f.uid, [])
|
||||
f.copies = copies.get(f.uid, 0)
|
||||
f.place = lines.get(f.uid, f.place)
|
||||
return found
|
||||
|
||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||
|
||||
Reference in new issue
Block a user