diff --git a/app/dupes.py b/app/dupes.py index 5c5cc31..5d35e64 100644 --- a/app/dupes.py +++ b/app/dupes.py @@ -17,14 +17,30 @@ rows that each match all the others, never a chain: a row with no place matches Belgium and the Netherlands, and must not make them one role. Each group keeps one copy (`keep_order`, smallest wins) and the rest point at -it. Pure by design: no store, no SQL, no network, so the rule is tested on +it. + +A second rule folds what the first leaves apart on purpose: one posting per +place. A host that lists one job once per office, each with its own id, gives +rows with the same company and title whose towns never meet. `fold` merges +those rows of one host into the kept role of the first group, when the texts +do not say they are different jobs (`similar`), and reports them in +`Folded.placed`: the store gives the kept role each such copy's place with the +copy's own link. This holds for every host, not only readers that declare +`READS.places == "copies"`: on the live database 1,739 company-and-title +groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and +Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts +share fewer than `SAME_TEXT` of their words, or without a link to keep, or +listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on the strings boards really write and the storage layer only feeds it rows. """ from __future__ import annotations +import re from collections.abc import Iterable from dataclasses import dataclass +from functools import lru_cache +from typing import NamedTuple from app import places from app.models import country_of @@ -48,6 +64,11 @@ class Copy: host: str has_description: bool posted_at: str | None + #: The start of the description, else the teaser, for `similar`; '' when + #: the row has neither. + text: str = "" + #: The link that opens this row on its host. + url: str = "" def _country(c: Copy) -> str: @@ -118,3 +139,78 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]: if row is not keeper: result[row.uid] = keeper.uid return result + + +#: The share of words two texts must have in common (Jaccard, over the first +#: 3,000 characters) for one job in two places. Measured on the live +#: database: copies of a posting share nearly all of it, two jobs that +#: happen to share a title share little more than boilerplate, and a +#: translation shares none, which keeps it apart. +SAME_TEXT = 0.3 + +TEXT_CHARS = 3000 + + +@lru_cache(maxsize=8192) +def _words(text: str) -> frozenset[str]: + return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower())) + + +def similar(a: str, b: str) -> bool: + """Whether two texts can be one job: a text that is missing says nothing + against it, else they share at least `SAME_TEXT` of their words.""" + wa, wb = _words(a), _words(b) + if not wa or not wb: + return True + return len(wa & wb) / len(wa | wb) >= SAME_TEXT + + +class Folded(NamedTuple): + """`assign`'s answer plus the rows that are one place of another role: + `placed` holds the uid of each copy that `fold` merged by place.""" + + dup_of: dict[str, str | None] + placed: frozenset[str] + + +def fold(rows: Iterable[Copy]) -> Folded: + """`assign`, then the per-place copies of each host folded in. + + Within a `(company_key, title_key)` the kept rows of `assign` are grouped + again, in `keep_order`: a row joins the first group whose members are all + of its host and `similar` to it. Only a row with a link can be folded, as + its link is what the kept role keeps. Every row of a merged group points + at its first row, the rows that pointed at a folded one included.""" + rows = list(rows) + result = assign(rows) + by_uid = {r.uid: r for r in rows} + parts: dict[tuple[str, str], list[Copy]] = {} + for uid, dup in result.items(): + row = by_uid[uid] + if dup is None and row.company_key and row.url: + parts.setdefault((row.company_key, row.title_key), []).append(row) + moved: dict[str, str] = {} + for part in parts.values(): + groups: list[list[Copy]] = [] + for row in sorted(part, key=keep_order): + for group in groups: + if all(m.host == row.host and similar(m.text, row.text) for m in group): + group.append(row) + break + else: + groups.append([row]) + for keeper, *others in groups: + moved.update((other.uid, keeper.uid) for other in others) + for uid, dup in result.items(): + if uid in moved: + result[uid] = moved[uid] + elif dup in moved: + result[uid] = moved[dup] + return Folded(result, frozenset(moved)) + + +def place_line(texts: Iterable[str]) -> str: + """The line a feed row shows for a role's places: the first, and "+N" for + the N others. Repeated texts count once.""" + found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip())) + return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found) diff --git a/app/static/app.css b/app/static/app.css index 9376299..ee87a56 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -942,6 +942,7 @@ body[data-reader="open"] .company-strip { display: none; } line-height: 1.4; } .also a b { display: block; color: var(--faint); font-size: .78rem; font-weight: 600; } +.also span { display: block; margin-bottom: .55rem; color: var(--dim); font-size: .92rem; line-height: 1.4; } @media (hover: hover) { .also a:hover { color: var(--fg); } } diff --git a/app/static/roles.js b/app/static/roles.js index 635c057..92f4d70 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -885,6 +885,42 @@ `; } + /* The role's places in the host's order, one entry for each place text a + link opens: the copies' own links, and the role's link for its own. */ + function placeEntries(r) { + const seen = new Set(); + const own = safeUrl(r.original_url); + const out = []; + for (const p of Array.isArray(r.places) ? r.places : []) { + const url = p.url ? safeUrl(p.url) : own; + const key = `${p.url || ""}\u0001${p.position}\u0001${p.text}`; + if (!p.text || seen.has(key)) continue; + seen.add(key); + out.push({ text: p.text, url }); + } + return out; + } + + /* The place line of the reader: the first place and "+N" for the others, + as the feed's row shows it. */ + function placeLine(r) { + const texts = [...new Set(placeEntries(r).map((e) => e.text))]; + return texts.length > 1 ? `${texts[0]} +${texts.length - 1}` : r.place || texts[0] || ""; + } + + /* Every place of a role that has several, each a link to its own posting, + laid out as "Also posted". */ + function placesList(r) { + const entries = placeEntries(r); + if (new Set(entries.map((e) => e.text)).size < 2) return ""; + return `
+

Places

+ ${entries.map((e) => e.url + ? `${esc(e.text)} ↗` + : `${esc(e.text)}`).join("")} +
`; + } + function drawRow(el, role) { const scored = role.score !== null && role.score !== undefined; const pct = scored ? Math.round(Math.min(1, Math.max(0, role.score)) * 100) : 0; @@ -1243,7 +1279,10 @@ : !loaded ? `

Loading…

` : text ? paragraphs(text) - : `

The board gave no description for this one.

`; + : (r.teaser || "").trim() + ? `

A short description from the board's + list; the board gave no full text.

${paragraphs(r.teaser.trim())}` + : `

The board gave no description for this one.

`; const pay = payDisplay(r); const careers = safeUrl(companies.get(r.company_key)?.careers_url); const waiting = state.summarising === state.readingUid; @@ -1264,7 +1303,7 @@ ${careers ? `Careers ↗` : ""} - ${r.place ? `${esc(r.place)}` : ""} + ${placeLine(r) ? `${esc(placeLine(r))}` : ""} ${r.first_seen ? `${ago(r.first_seen)}` : ""} ${badges(r)} ${tagChips(r.tags)} @@ -1287,6 +1326,7 @@
${body}
${out ? `

Read it on ${esc(host(out) || "the board")} ↗

` : ""} + ${loaded ? placesList(r) : ""} ${loaded ? alsoPosted(r) : ""}
diff --git a/app/storage/store.py b/app/storage/store.py index 4f8d184..8a07ade 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -10,7 +10,7 @@ from __future__ import annotations import json import sqlite3 -from collections.abc import Callable, Iterator, MutableMapping, Sequence +from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence from contextlib import closing, contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path @@ -281,6 +281,13 @@ def _int_or_none(value: Any) -> int | None: _HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')" +#: A copy that is not one place of its kept role: the copies a role lists +#: under "Also posted" and counts as "x N". `{kept}` is the kept role's uid +#: as SQL, and a place copy is the one whose link the kept role holds. +_NOT_A_PLACE = ("original_url NOT IN (SELECT url FROM role_places " + "WHERE uid = {kept} AND url != '')") + + class DupeCounts(NamedTuple): """What `rebuild_dupes` left and did: kept roles with at least one copy, open roles that are a copy, and rows whose `dup_of` it rewrote.""" @@ -290,8 +297,35 @@ class DupeCounts(NamedTuple): changed: int +def _assign_only(rows: Iterable[dupes.Copy]) -> dupes.Folded: + return dupes.Folded(dupes.assign(rows), frozenset()) + + +def _sync_copy_places(conn: sqlite3.Connection, placed: dict[str, str | None]) -> None: + """Make the `role_places` rows with a `url` those of the copies `placed` + (`copy uid -> kept uid`): each own place of a copy, on the kept role, + with the copy's link. Written only when they differ from what is stored, + so an unchanged database is not touched. A copy that closed, or a group + that dissolved, is no longer in `placed`, and its rows go.""" + want: set[tuple[str, int, str, str, int | None, str]] = set() + if placed: + marks = ",".join("?" * len(placed)) + for uid, pos, text, fold, place_id, url in conn.execute( + f"""SELECT rp.uid, rp.position, rp.text, rp.fold, rp.place_id, r.original_url + FROM role_places rp JOIN roles r ON r.uid = rp.uid + WHERE rp.url = '' AND rp.uid IN ({marks})""", list(placed)): + want.add((str(placed[uid]), pos, text, fold, place_id, url)) + have = set(conn.execute( + "SELECT uid, position, text, fold, place_id, url FROM role_places WHERE url != ''")) + if want != have: + conn.execute("DELETE FROM role_places WHERE url != ''") + conn.executemany( + "INSERT INTO role_places (uid, position, text, fold, place_id, url) " + "VALUES (?, ?, ?, ?, ?, ?)", sorted(want, key=lambda r: (r[0], r[5], r[1], r[2]))) + + def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts: - """Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside + """Recompute `roles.dup_of` from scratch with `app.dupes.fold`, inside the caller's transaction: `Store.rebuild_dupes` and migration 18. Only the open roles of a `(company_key, title_key)` with more than one @@ -309,26 +343,40 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts: them moves to it unless that agent already voted on it; a vote that cannot move stays where it is. An unchanged group is not touched, so marking a kept role unread again sticks. Applications keep their - `role_uid`, and scores of a copy go at the agent's next `score`.""" + `role_uid`, and scores of a copy go at the agent's next `score`. + + A copy the host published for another place (`dupes.Folded.placed`) + leaves its places on the kept role as `role_places` rows with the copy's + link, see `_sync_copy_places`.""" + # Migration 18 runs this on a file that has no `teaser` and no + # `role_places` yet: there it groups as before and folds no places. + current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)")) + text = "COALESCE(NULLIF(description, ''), teaser)" if current else "description" rows = conn.execute( f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host, - description != '' AS described, posted_at + description != '' AS described, posted_at, + SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url FROM roles WHERE closed_at IS NULL AND company_key != '' AND (company_key, title_key) IN ( SELECT company_key, title_key FROM roles WHERE closed_at IS NULL AND company_key != '' GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall() - new = dupes.assign( + made = dupes.fold if current else _assign_only + folded = made( dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4], - host=r[5], has_description=bool(r[6]), posted_at=r[7] or None) + host=r[5], has_description=bool(r[6]), posted_at=r[7] or None, + text=r[8] or "", url=r[9] or "") for r in rows) + new = folded.dup_of old: dict[str, str | None] = dict.fromkeys(new) old.update((r[0], r[1]) for r in conn.execute( "SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL")) updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was] conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates) + if current: + _sync_copy_places(conn, {uid: new[uid] for uid in folded.placed}) changed = {uid for _, uid in updates} members: dict[str, set[str]] = {} for uid, dup in new.items(): @@ -707,7 +755,8 @@ class Store: or a role without any. See `Role.copies`.""" return [RoleCopy.from_row(r) for r in self._rows( f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles - WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))] + WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")} + ORDER BY posted_at, uid""", (uid, uid))] def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]: """`uid -> open copies` for a page of kept roles, one grouped query @@ -720,8 +769,25 @@ class Store: return {str(r["dup_of"]): int(r["n"]) for r in self._rows( "SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of " "WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL " + f"AND {_NOT_A_PLACE.format(kept='dup_of')} " "GROUP BY dup_of", (json.dumps(list(uids)),))} + def _place_lines(self, uids: Sequence[str]) -> dict[str, str]: + """`uid -> the line the feed shows` for a page of roles that have + more than one place (`dupes.place_line`); a role with one place or + none is absent and keeps `roles.place`.""" + if not uids: + return {} + found: dict[str, list[str]] = {} + for r in self._rows( + """SELECT uid, text FROM role_places + WHERE uid IN (SELECT value FROM json_each(?)) + ORDER BY uid, url != '', url, position, rowid""", + (json.dumps(list(uids)),)): + found.setdefault(str(r["uid"]), []).append(str(r["text"])) + return {uid: dupes.place_line(texts) for uid, texts in found.items() + if len(set(texts)) > 1} + def rebuild_dupes(self) -> DupeCounts: """Recompute which open roles are copies of one posting, in one transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE` @@ -966,9 +1032,11 @@ class Store: found.append(RoleRow.from_row(row)) by_uid = self._tags_by_uid([f.uid for f in found]) copies = self._copy_counts([f.uid for f in found]) + lines = self._place_lines([f.uid for f in found]) for f in found: f.tags = by_uid.get(f.uid, []) f.copies = copies.get(f.uid, 0) + f.place = lines.get(f.uid, f.place) return found def count_roles(self, company: str | None = None, place: str | None = None, diff --git a/tests/test_dupes.py b/tests/test_dupes.py index f3f6c22..bcd00a0 100644 --- a/tests/test_dupes.py +++ b/tests/test_dupes.py @@ -12,7 +12,15 @@ if str(ROOT) not in sys.path: import pytest # noqa: E402 -from app.dupes import Copy, assign, keep_order, same_posting # noqa: E402 +from app.dupes import ( # noqa: E402 + Copy, + assign, + fold, + keep_order, + place_line, + same_posting, + similar, +) from app.places import towns # noqa: E402 @@ -194,3 +202,42 @@ def test_a_row_without_a_place_never_bridges_two_countries(): assert got == {"a": None, "b": "a", "c": "a", "d": None, "e": None} assert not same_posting(cp("a", country="EU", place="Belgium"), cp("b", country="EU", place="Netherlands")) + + +def pc(uid: str, place: str, text: str = "", host: str = "greenhouse", url: str = "u", + title: str = "engineer") -> Copy: + return Copy(uid, "acme", title, "DE", place, host, bool(text), None, text, url or "") + + +def test_one_posting_per_place_folds_into_the_first_and_only_within_a_host() -> None: + text = "We build the payment platform for our bank customers in Kotlin." + got = fold([pc("b", "Berlin", text), pc("m", "Munich", text), pc("h", "Hamburg", text), + pc("x", "Kiel", text, host="lever")]) + assert got.dup_of == {"b": None, "m": "b", "h": "b", "x": None} + assert got.placed == {"m", "h"} + + +def test_folding_needs_the_same_title_a_link_and_texts_that_can_be_one_job() -> None: + a = "Build payment services in Kotlin for our bank customers and their teams." + b = "Negotiate enterprise cloud contracts with procurement and legal buyers." + got = fold([pc("1", "Berlin", a), pc("2", "Munich", b), pc("3", "Hamburg", a, url=""), + pc("4", "Kiel", a, title="manager")]) + assert got.dup_of == {"1": None, "2": None, "3": None, "4": None} + assert fold([pc("1", "Berlin", a), pc("2", "Munich", "")]).placed == {"2"} + + +def test_a_copy_of_a_folded_row_follows_it_to_the_new_kept_role() -> None: + """Berlin is kept over Munich; a second Munich row that was Munich's copy + now points at Berlin, and is not a place of its own.""" + text = "We build the payment platform for our bank customers in Kotlin." + same = fold([pc("b", "Berlin", text), pc("m", "Munich", text), + pc("m2", "Munich, Germany", text)]) + assert same.dup_of == {"b": None, "m": "b", "m2": "b"} + assert same.placed == {"m"} # m2 is m's own copy, not another place + + +def test_similar_and_place_line() -> None: + assert similar("", "anything at all") and similar("a job in Kotlin", "a job in Kotlin") + assert not similar("sell cloud contracts to buyers", "build kotlin payment services") + assert place_line(["Berlin", "Munich", "Berlin", "Hamburg"]) == "Berlin +2" + assert place_line(["Berlin", " "]) == "Berlin" and place_line([]) == "" diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py index 1c8cbc1..700ec28 100644 --- a/tests/test_dupes_storage.py +++ b/tests/test_dupes_storage.py @@ -37,9 +37,9 @@ def dup_of(store: Store) -> dict[str, str | None]: def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart( store: Store) -> None: """SAP's repeats: three Garching rows are one kept role (the one with - the description) and two copies; New York is its own role.""" + the description) and two copies; New York, listed by another host, is its own role.""" garching = [role("1", body=""), role("2"), role("3", body="")] - york = role("4", place="New York, NY, US") + york = role("4", place="New York, NY, US", board="greenhouse") for item in (*garching, york): store.upsert_role(item) @@ -61,12 +61,80 @@ def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store) assert dup_of(store) == {board.uid: None, search.uid: board.uid} -def test_berlin_and_munich_stay_two_roles(store: Store) -> None: - for item in (role("b", place="Berlin"), role("m", place="Munich")): +def kept_of(store: Store, *items: Item) -> str: + """The uid of the one of these roles that is not a copy.""" + found = dup_of(store) + (kept,) = [i.uid for i in items if found[i.uid] is None] + return kept + + +def test_berlin_and_munich_of_one_host_fold_into_one_role_with_two_places( + store: Store) -> None: + """One posting per place on one host is one role that lists both places, + each with its own link; the count of copies does not show it as `x2`.""" + berlin, munich = role("b", place="Berlin"), role("m", place="Munich") + for item in (berlin, munich): + store.upsert_role(item) + assert store.rebuild_dupes() == (1, 1, 1) + kept = kept_of(store, berlin, munich) + + got = store.role(kept) + assert got is not None + assert sorted(p.text for p in got.places) == ["Berlin", "Munich"] + assert sorted(p.url for p in got.places if p.url) in ( + ["https://jobs.example/b"], ["https://jobs.example/m"]) + assert got.copies == [] + (row,) = store.roles_page() + assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0) + + +def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None: + a = role("a", place="Berlin", body="Build payment services in Kotlin for our bank " * 5) + b = role("b", place="Munich", body="Sell cloud contracts to enterprise buyers " * 5) + c = role("c", place="Hamburg", board="greenhouse", body=a.body) + for item in (a, b, c): store.upsert_role(item) assert store.rebuild_dupes() == (0, 0, 0) +def own_and_copy_places(store: Store, uid: str) -> list[tuple[str, bool]]: + got = store.role(uid) + assert got is not None + return sorted((p.text, bool(p.url)) for p in got.places) + + +def test_a_place_copy_closing_or_its_board_off_takes_its_place_away(store: Store) -> None: + from app.jobs.sources import set_fetch + sub = store.add_subscription("ats", "successfactors:sap", label="SAP") + items = [role("b", place="Berlin"), role("m", place="Munich"), role("k", place="Kiel")] + for item in items: + store.upsert_role(item, sub) + store.rebuild_dupes() + kept = kept_of(store, *items) + assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Kiel", "Munich"] + + closing = next(i for i in items if i.uid != kept) + with sqlite3.connect(store.path) as conn: + conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (closing.uid,)) + store.rebuild_dupes() + assert len(own_and_copy_places(store, kept)) == 2 + assert store.rebuild_dupes().changed == 0 + + set_fetch(store, sub.id, False) + assert set(dup_of(store).values()) == {None} + assert not any(url for _, url in own_and_copy_places(store, kept)) + + +def test_a_repull_keeps_the_copies_places(store: Store) -> None: + items = [role("b", place="Berlin"), role("m", place="Munich")] + for item in items: + store.upsert_role(item) + store.rebuild_dupes() + kept = kept_of(store, *items) + store.upsert_role(next(i for i in items if i.uid == kept)) + assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Munich"] + + def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared( store: Store) -> None: first, second, third = role("1", day=18), role("2", day=19), role("3", day=20) @@ -129,7 +197,7 @@ def test_a_closed_kept_role_hands_its_read_and_vote_to_the_next(store: Store) -> def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None: - kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin") + kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin", board="greenhouse") for item in (kept, copy, other): store.upsert_role(item) store.set_tags(item.uid, ["python"])