From 3d1fed7c0e86cba66ec200215a72cc821b1a1533 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 09:48:49 +0200 Subject: [PATCH] Fold per-place copies of one host into one role that lists every place with its own link The reader shows every place, each linked to its copy, and the teaser marked as a short description when there is no text. Co-Authored-By: Claude Opus 5.5 --- app/dupes.py | 98 ++++++++++++++++++++++++++++++++++++- app/static/app.css | 1 + app/static/roles.js | 44 ++++++++++++++++- app/storage/store.py | 82 ++++++++++++++++++++++++++++--- tests/test_dupes.py | 49 ++++++++++++++++++- tests/test_dupes_storage.py | 78 +++++++++++++++++++++++++++-- 6 files changed, 336 insertions(+), 16 deletions(-) diff --git a/app/dupes.py b/app/dupes.py index 5c5cc31..5d35e64 100644 --- a/app/dupes.py +++ b/app/dupes.py @@ -17,14 +17,30 @@ rows that each match all the others, never a chain: a row with no place matches Belgium and the Netherlands, and must not make them one role. Each group keeps one copy (`keep_order`, smallest wins) and the rest point at -it. Pure by design: no store, no SQL, no network, so the rule is tested on +it. + +A second rule folds what the first leaves apart on purpose: one posting per +place. A host that lists one job once per office, each with its own id, gives +rows with the same company and title whose towns never meet. `fold` merges +those rows of one host into the kept role of the first group, when the texts +do not say they are different jobs (`similar`), and reports them in +`Folded.placed`: the store gives the kept role each such copy's place with the +copy's own link. This holds for every host, not only readers that declare +`READS.places == "copies"`: on the live database 1,739 company-and-title +groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and +Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts +share fewer than `SAME_TEXT` of their words, or without a link to keep, or +listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on the strings boards really write and the storage layer only feeds it rows. """ from __future__ import annotations +import re from collections.abc import Iterable from dataclasses import dataclass +from functools import lru_cache +from typing import NamedTuple from app import places from app.models import country_of @@ -48,6 +64,11 @@ class Copy: host: str has_description: bool posted_at: str | None + #: The start of the description, else the teaser, for `similar`; '' when + #: the row has neither. + text: str = "" + #: The link that opens this row on its host. + url: str = "" def _country(c: Copy) -> str: @@ -118,3 +139,78 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]: if row is not keeper: result[row.uid] = keeper.uid return result + + +#: The share of words two texts must have in common (Jaccard, over the first +#: 3,000 characters) for one job in two places. Measured on the live +#: database: copies of a posting share nearly all of it, two jobs that +#: happen to share a title share little more than boilerplate, and a +#: translation shares none, which keeps it apart. +SAME_TEXT = 0.3 + +TEXT_CHARS = 3000 + + +@lru_cache(maxsize=8192) +def _words(text: str) -> frozenset[str]: + return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower())) + + +def similar(a: str, b: str) -> bool: + """Whether two texts can be one job: a text that is missing says nothing + against it, else they share at least `SAME_TEXT` of their words.""" + wa, wb = _words(a), _words(b) + if not wa or not wb: + return True + return len(wa & wb) / len(wa | wb) >= SAME_TEXT + + +class Folded(NamedTuple): + """`assign`'s answer plus the rows that are one place of another role: + `placed` holds the uid of each copy that `fold` merged by place.""" + + dup_of: dict[str, str | None] + placed: frozenset[str] + + +def fold(rows: Iterable[Copy]) -> Folded: + """`assign`, then the per-place copies of each host folded in. + + Within a `(company_key, title_key)` the kept rows of `assign` are grouped + again, in `keep_order`: a row joins the first group whose members are all + of its host and `similar` to it. Only a row with a link can be folded, as + its link is what the kept role keeps. Every row of a merged group points + at its first row, the rows that pointed at a folded one included.""" + rows = list(rows) + result = assign(rows) + by_uid = {r.uid: r for r in rows} + parts: dict[tuple[str, str], list[Copy]] = {} + for uid, dup in result.items(): + row = by_uid[uid] + if dup is None and row.company_key and row.url: + parts.setdefault((row.company_key, row.title_key), []).append(row) + moved: dict[str, str] = {} + for part in parts.values(): + groups: list[list[Copy]] = [] + for row in sorted(part, key=keep_order): + for group in groups: + if all(m.host == row.host and similar(m.text, row.text) for m in group): + group.append(row) + break + else: + groups.append([row]) + for keeper, *others in groups: + moved.update((other.uid, keeper.uid) for other in others) + for uid, dup in result.items(): + if uid in moved: + result[uid] = moved[uid] + elif dup in moved: + result[uid] = moved[dup] + return Folded(result, frozenset(moved)) + + +def place_line(texts: Iterable[str]) -> str: + """The line a feed row shows for a role's places: the first, and "+N" for + the N others. Repeated texts count once.""" + found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip())) + return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found) diff --git a/app/static/app.css b/app/static/app.css index 9376299..ee87a56 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -942,6 +942,7 @@ body[data-reader="open"] .company-strip { display: none; } line-height: 1.4; } .also a b { display: block; color: var(--faint); font-size: .78rem; font-weight: 600; } +.also span { display: block; margin-bottom: .55rem; color: var(--dim); font-size: .92rem; line-height: 1.4; } @media (hover: hover) { .also a:hover { color: var(--fg); } } diff --git a/app/static/roles.js b/app/static/roles.js index 635c057..92f4d70 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -885,6 +885,42 @@ `; } + /* The role's places in the host's order, one entry for each place text a + link opens: the copies' own links, and the role's link for its own. */ + function placeEntries(r) { + const seen = new Set(); + const own = safeUrl(r.original_url); + const out = []; + for (const p of Array.isArray(r.places) ? r.places : []) { + const url = p.url ? safeUrl(p.url) : own; + const key = `${p.url || ""}\u0001${p.position}\u0001${p.text}`; + if (!p.text || seen.has(key)) continue; + seen.add(key); + out.push({ text: p.text, url }); + } + return out; + } + + /* The place line of the reader: the first place and "+N" for the others, + as the feed's row shows it. */ + function placeLine(r) { + const texts = [...new Set(placeEntries(r).map((e) => e.text))]; + return texts.length > 1 ? `${texts[0]} +${texts.length - 1}` : r.place || texts[0] || ""; + } + + /* Every place of a role that has several, each a link to its own posting, + laid out as "Also posted". */ + function placesList(r) { + const entries = placeEntries(r); + if (new Set(entries.map((e) => e.text)).size < 2) return ""; + return `
+

Places

+ ${entries.map((e) => e.url + ? `${esc(e.text)} ↗` + : `${esc(e.text)}`).join("")} +
`; + } + function drawRow(el, role) { const scored = role.score !== null && role.score !== undefined; const pct = scored ? Math.round(Math.min(1, Math.max(0, role.score)) * 100) : 0; @@ -1243,7 +1279,10 @@ : !loaded ? `

Loading…

` : text ? paragraphs(text) - : `

The board gave no description for this one.

`; + : (r.teaser || "").trim() + ? `

A short description from the board's + list; the board gave no full text.

${paragraphs(r.teaser.trim())}` + : `

The board gave no description for this one.

`; const pay = payDisplay(r); const careers = safeUrl(companies.get(r.company_key)?.careers_url); const waiting = state.summarising === state.readingUid; @@ -1264,7 +1303,7 @@ ${careers ? `Careers ↗` : ""} - ${r.place ? `${esc(r.place)}` : ""} + ${placeLine(r) ? `${esc(placeLine(r))}` : ""} ${r.first_seen ? `${ago(r.first_seen)}` : ""} ${badges(r)} ${tagChips(r.tags)} @@ -1287,6 +1326,7 @@
${body}
${out ? `

Read it on ${esc(host(out) || "the board")} ↗

` : ""} + ${loaded ? placesList(r) : ""} ${loaded ? alsoPosted(r) : ""}
diff --git a/app/storage/store.py b/app/storage/store.py index 4f8d184..8a07ade 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -10,7 +10,7 @@ from __future__ import annotations import json import sqlite3 -from collections.abc import Callable, Iterator, MutableMapping, Sequence +from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence from contextlib import closing, contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path @@ -281,6 +281,13 @@ def _int_or_none(value: Any) -> int | None: _HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')" +#: A copy that is not one place of its kept role: the copies a role lists +#: under "Also posted" and counts as "x N". `{kept}` is the kept role's uid +#: as SQL, and a place copy is the one whose link the kept role holds. +_NOT_A_PLACE = ("original_url NOT IN (SELECT url FROM role_places " + "WHERE uid = {kept} AND url != '')") + + class DupeCounts(NamedTuple): """What `rebuild_dupes` left and did: kept roles with at least one copy, open roles that are a copy, and rows whose `dup_of` it rewrote.""" @@ -290,8 +297,35 @@ class DupeCounts(NamedTuple): changed: int +def _assign_only(rows: Iterable[dupes.Copy]) -> dupes.Folded: + return dupes.Folded(dupes.assign(rows), frozenset()) + + +def _sync_copy_places(conn: sqlite3.Connection, placed: dict[str, str | None]) -> None: + """Make the `role_places` rows with a `url` those of the copies `placed` + (`copy uid -> kept uid`): each own place of a copy, on the kept role, + with the copy's link. Written only when they differ from what is stored, + so an unchanged database is not touched. A copy that closed, or a group + that dissolved, is no longer in `placed`, and its rows go.""" + want: set[tuple[str, int, str, str, int | None, str]] = set() + if placed: + marks = ",".join("?" * len(placed)) + for uid, pos, text, fold, place_id, url in conn.execute( + f"""SELECT rp.uid, rp.position, rp.text, rp.fold, rp.place_id, r.original_url + FROM role_places rp JOIN roles r ON r.uid = rp.uid + WHERE rp.url = '' AND rp.uid IN ({marks})""", list(placed)): + want.add((str(placed[uid]), pos, text, fold, place_id, url)) + have = set(conn.execute( + "SELECT uid, position, text, fold, place_id, url FROM role_places WHERE url != ''")) + if want != have: + conn.execute("DELETE FROM role_places WHERE url != ''") + conn.executemany( + "INSERT INTO role_places (uid, position, text, fold, place_id, url) " + "VALUES (?, ?, ?, ?, ?, ?)", sorted(want, key=lambda r: (r[0], r[5], r[1], r[2]))) + + def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts: - """Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside + """Recompute `roles.dup_of` from scratch with `app.dupes.fold`, inside the caller's transaction: `Store.rebuild_dupes` and migration 18. Only the open roles of a `(company_key, title_key)` with more than one @@ -309,26 +343,40 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts: them moves to it unless that agent already voted on it; a vote that cannot move stays where it is. An unchanged group is not touched, so marking a kept role unread again sticks. Applications keep their - `role_uid`, and scores of a copy go at the agent's next `score`.""" + `role_uid`, and scores of a copy go at the agent's next `score`. + + A copy the host published for another place (`dupes.Folded.placed`) + leaves its places on the kept role as `role_places` rows with the copy's + link, see `_sync_copy_places`.""" + # Migration 18 runs this on a file that has no `teaser` and no + # `role_places` yet: there it groups as before and folds no places. + current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)")) + text = "COALESCE(NULLIF(description, ''), teaser)" if current else "description" rows = conn.execute( f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host, - description != '' AS described, posted_at + description != '' AS described, posted_at, + SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url FROM roles WHERE closed_at IS NULL AND company_key != '' AND (company_key, title_key) IN ( SELECT company_key, title_key FROM roles WHERE closed_at IS NULL AND company_key != '' GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall() - new = dupes.assign( + made = dupes.fold if current else _assign_only + folded = made( dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4], - host=r[5], has_description=bool(r[6]), posted_at=r[7] or None) + host=r[5], has_description=bool(r[6]), posted_at=r[7] or None, + text=r[8] or "", url=r[9] or "") for r in rows) + new = folded.dup_of old: dict[str, str | None] = dict.fromkeys(new) old.update((r[0], r[1]) for r in conn.execute( "SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL")) updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was] conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates) + if current: + _sync_copy_places(conn, {uid: new[uid] for uid in folded.placed}) changed = {uid for _, uid in updates} members: dict[str, set[str]] = {} for uid, dup in new.items(): @@ -707,7 +755,8 @@ class Store: or a role without any. See `Role.copies`.""" return [RoleCopy.from_row(r) for r in self._rows( f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles - WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))] + WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")} + ORDER BY posted_at, uid""", (uid, uid))] def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]: """`uid -> open copies` for a page of kept roles, one grouped query @@ -720,8 +769,25 @@ class Store: return {str(r["dup_of"]): int(r["n"]) for r in self._rows( "SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of " "WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL " + f"AND {_NOT_A_PLACE.format(kept='dup_of')} " "GROUP BY dup_of", (json.dumps(list(uids)),))} + def _place_lines(self, uids: Sequence[str]) -> dict[str, str]: + """`uid -> the line the feed shows` for a page of roles that have + more than one place (`dupes.place_line`); a role with one place or + none is absent and keeps `roles.place`.""" + if not uids: + return {} + found: dict[str, list[str]] = {} + for r in self._rows( + """SELECT uid, text FROM role_places + WHERE uid IN (SELECT value FROM json_each(?)) + ORDER BY uid, url != '', url, position, rowid""", + (json.dumps(list(uids)),)): + found.setdefault(str(r["uid"]), []).append(str(r["text"])) + return {uid: dupes.place_line(texts) for uid, texts in found.items() + if len(set(texts)) > 1} + def rebuild_dupes(self) -> DupeCounts: """Recompute which open roles are copies of one posting, in one transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE` @@ -966,9 +1032,11 @@ class Store: found.append(RoleRow.from_row(row)) by_uid = self._tags_by_uid([f.uid for f in found]) copies = self._copy_counts([f.uid for f in found]) + lines = self._place_lines([f.uid for f in found]) for f in found: f.tags = by_uid.get(f.uid, []) f.copies = copies.get(f.uid, 0) + f.place = lines.get(f.uid, f.place) return found def count_roles(self, company: str | None = None, place: str | None = None, diff --git a/tests/test_dupes.py b/tests/test_dupes.py index f3f6c22..bcd00a0 100644 --- a/tests/test_dupes.py +++ b/tests/test_dupes.py @@ -12,7 +12,15 @@ if str(ROOT) not in sys.path: import pytest # noqa: E402 -from app.dupes import Copy, assign, keep_order, same_posting # noqa: E402 +from app.dupes import ( # noqa: E402 + Copy, + assign, + fold, + keep_order, + place_line, + same_posting, + similar, +) from app.places import towns # noqa: E402 @@ -194,3 +202,42 @@ def test_a_row_without_a_place_never_bridges_two_countries(): assert got == {"a": None, "b": "a", "c": "a", "d": None, "e": None} assert not same_posting(cp("a", country="EU", place="Belgium"), cp("b", country="EU", place="Netherlands")) + + +def pc(uid: str, place: str, text: str = "", host: str = "greenhouse", url: str = "u", + title: str = "engineer") -> Copy: + return Copy(uid, "acme", title, "DE", place, host, bool(text), None, text, url or "") + + +def test_one_posting_per_place_folds_into_the_first_and_only_within_a_host() -> None: + text = "We build the payment platform for our bank customers in Kotlin." + got = fold([pc("b", "Berlin", text), pc("m", "Munich", text), pc("h", "Hamburg", text), + pc("x", "Kiel", text, host="lever")]) + assert got.dup_of == {"b": None, "m": "b", "h": "b", "x": None} + assert got.placed == {"m", "h"} + + +def test_folding_needs_the_same_title_a_link_and_texts_that_can_be_one_job() -> None: + a = "Build payment services in Kotlin for our bank customers and their teams." + b = "Negotiate enterprise cloud contracts with procurement and legal buyers." + got = fold([pc("1", "Berlin", a), pc("2", "Munich", b), pc("3", "Hamburg", a, url=""), + pc("4", "Kiel", a, title="manager")]) + assert got.dup_of == {"1": None, "2": None, "3": None, "4": None} + assert fold([pc("1", "Berlin", a), pc("2", "Munich", "")]).placed == {"2"} + + +def test_a_copy_of_a_folded_row_follows_it_to_the_new_kept_role() -> None: + """Berlin is kept over Munich; a second Munich row that was Munich's copy + now points at Berlin, and is not a place of its own.""" + text = "We build the payment platform for our bank customers in Kotlin." + same = fold([pc("b", "Berlin", text), pc("m", "Munich", text), + pc("m2", "Munich, Germany", text)]) + assert same.dup_of == {"b": None, "m": "b", "m2": "b"} + assert same.placed == {"m"} # m2 is m's own copy, not another place + + +def test_similar_and_place_line() -> None: + assert similar("", "anything at all") and similar("a job in Kotlin", "a job in Kotlin") + assert not similar("sell cloud contracts to buyers", "build kotlin payment services") + assert place_line(["Berlin", "Munich", "Berlin", "Hamburg"]) == "Berlin +2" + assert place_line(["Berlin", " "]) == "Berlin" and place_line([]) == "" diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py index 1c8cbc1..700ec28 100644 --- a/tests/test_dupes_storage.py +++ b/tests/test_dupes_storage.py @@ -37,9 +37,9 @@ def dup_of(store: Store) -> dict[str, str | None]: def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart( store: Store) -> None: """SAP's repeats: three Garching rows are one kept role (the one with - the description) and two copies; New York is its own role.""" + the description) and two copies; New York, listed by another host, is its own role.""" garching = [role("1", body=""), role("2"), role("3", body="")] - york = role("4", place="New York, NY, US") + york = role("4", place="New York, NY, US", board="greenhouse") for item in (*garching, york): store.upsert_role(item) @@ -61,12 +61,80 @@ def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store) assert dup_of(store) == {board.uid: None, search.uid: board.uid} -def test_berlin_and_munich_stay_two_roles(store: Store) -> None: - for item in (role("b", place="Berlin"), role("m", place="Munich")): +def kept_of(store: Store, *items: Item) -> str: + """The uid of the one of these roles that is not a copy.""" + found = dup_of(store) + (kept,) = [i.uid for i in items if found[i.uid] is None] + return kept + + +def test_berlin_and_munich_of_one_host_fold_into_one_role_with_two_places( + store: Store) -> None: + """One posting per place on one host is one role that lists both places, + each with its own link; the count of copies does not show it as `x2`.""" + berlin, munich = role("b", place="Berlin"), role("m", place="Munich") + for item in (berlin, munich): + store.upsert_role(item) + assert store.rebuild_dupes() == (1, 1, 1) + kept = kept_of(store, berlin, munich) + + got = store.role(kept) + assert got is not None + assert sorted(p.text for p in got.places) == ["Berlin", "Munich"] + assert sorted(p.url for p in got.places if p.url) in ( + ["https://jobs.example/b"], ["https://jobs.example/m"]) + assert got.copies == [] + (row,) = store.roles_page() + assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0) + + +def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None: + a = role("a", place="Berlin", body="Build payment services in Kotlin for our bank " * 5) + b = role("b", place="Munich", body="Sell cloud contracts to enterprise buyers " * 5) + c = role("c", place="Hamburg", board="greenhouse", body=a.body) + for item in (a, b, c): store.upsert_role(item) assert store.rebuild_dupes() == (0, 0, 0) +def own_and_copy_places(store: Store, uid: str) -> list[tuple[str, bool]]: + got = store.role(uid) + assert got is not None + return sorted((p.text, bool(p.url)) for p in got.places) + + +def test_a_place_copy_closing_or_its_board_off_takes_its_place_away(store: Store) -> None: + from app.jobs.sources import set_fetch + sub = store.add_subscription("ats", "successfactors:sap", label="SAP") + items = [role("b", place="Berlin"), role("m", place="Munich"), role("k", place="Kiel")] + for item in items: + store.upsert_role(item, sub) + store.rebuild_dupes() + kept = kept_of(store, *items) + assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Kiel", "Munich"] + + closing = next(i for i in items if i.uid != kept) + with sqlite3.connect(store.path) as conn: + conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (closing.uid,)) + store.rebuild_dupes() + assert len(own_and_copy_places(store, kept)) == 2 + assert store.rebuild_dupes().changed == 0 + + set_fetch(store, sub.id, False) + assert set(dup_of(store).values()) == {None} + assert not any(url for _, url in own_and_copy_places(store, kept)) + + +def test_a_repull_keeps_the_copies_places(store: Store) -> None: + items = [role("b", place="Berlin"), role("m", place="Munich")] + for item in items: + store.upsert_role(item) + store.rebuild_dupes() + kept = kept_of(store, *items) + store.upsert_role(next(i for i in items if i.uid == kept)) + assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Munich"] + + def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared( store: Store) -> None: first, second, third = role("1", day=18), role("2", day=19), role("3", day=20) @@ -129,7 +197,7 @@ def test_a_closed_kept_role_hands_its_read_and_vote_to_the_next(store: Store) -> def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None: - kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin") + kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin", board="greenhouse") for item in (kept, copy, other): store.upsert_role(item) store.set_tags(item.uid, ["python"])