diff --git a/app/dupes.py b/app/dupes.py index 5c5cc31..5d35e64 100644 --- a/app/dupes.py +++ b/app/dupes.py @@ -17,14 +17,30 @@ rows that each match all the others, never a chain: a row with no place matches Belgium and the Netherlands, and must not make them one role. Each group keeps one copy (`keep_order`, smallest wins) and the rest point at -it. Pure by design: no store, no SQL, no network, so the rule is tested on +it. + +A second rule folds what the first leaves apart on purpose: one posting per +place. A host that lists one job once per office, each with its own id, gives +rows with the same company and title whose towns never meet. `fold` merges +those rows of one host into the kept role of the first group, when the texts +do not say they are different jobs (`similar`), and reports them in +`Folded.placed`: the store gives the kept role each such copy's place with the +copy's own link. This holds for every host, not only readers that declare +`READS.places == "copies"`: on the live database 1,739 company-and-title +groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and +Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts +share fewer than `SAME_TEXT` of their words, or without a link to keep, or +listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on the strings boards really write and the storage layer only feeds it rows. """ from __future__ import annotations +import re from collections.abc import Iterable from dataclasses import dataclass +from functools import lru_cache +from typing import NamedTuple from app import places from app.models import country_of @@ -48,6 +64,11 @@ class Copy: host: str has_description: bool posted_at: str | None + #: The start of the description, else the teaser, for `similar`; '' when + #: the row has neither. + text: str = "" + #: The link that opens this row on its host. + url: str = "" def _country(c: Copy) -> str: @@ -118,3 +139,78 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]: if row is not keeper: result[row.uid] = keeper.uid return result + + +#: The share of words two texts must have in common (Jaccard, over the first +#: 3,000 characters) for one job in two places. Measured on the live +#: database: copies of a posting share nearly all of it, two jobs that +#: happen to share a title share little more than boilerplate, and a +#: translation shares none, which keeps it apart. +SAME_TEXT = 0.3 + +TEXT_CHARS = 3000 + + +@lru_cache(maxsize=8192) +def _words(text: str) -> frozenset[str]: + return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower())) + + +def similar(a: str, b: str) -> bool: + """Whether two texts can be one job: a text that is missing says nothing + against it, else they share at least `SAME_TEXT` of their words.""" + wa, wb = _words(a), _words(b) + if not wa or not wb: + return True + return len(wa & wb) / len(wa | wb) >= SAME_TEXT + + +class Folded(NamedTuple): + """`assign`'s answer plus the rows that are one place of another role: + `placed` holds the uid of each copy that `fold` merged by place.""" + + dup_of: dict[str, str | None] + placed: frozenset[str] + + +def fold(rows: Iterable[Copy]) -> Folded: + """`assign`, then the per-place copies of each host folded in. + + Within a `(company_key, title_key)` the kept rows of `assign` are grouped + again, in `keep_order`: a row joins the first group whose members are all + of its host and `similar` to it. Only a row with a link can be folded, as + its link is what the kept role keeps. Every row of a merged group points + at its first row, the rows that pointed at a folded one included.""" + rows = list(rows) + result = assign(rows) + by_uid = {r.uid: r for r in rows} + parts: dict[tuple[str, str], list[Copy]] = {} + for uid, dup in result.items(): + row = by_uid[uid] + if dup is None and row.company_key and row.url: + parts.setdefault((row.company_key, row.title_key), []).append(row) + moved: dict[str, str] = {} + for part in parts.values(): + groups: list[list[Copy]] = [] + for row in sorted(part, key=keep_order): + for group in groups: + if all(m.host == row.host and similar(m.text, row.text) for m in group): + group.append(row) + break + else: + groups.append([row]) + for keeper, *others in groups: + moved.update((other.uid, keeper.uid) for other in others) + for uid, dup in result.items(): + if uid in moved: + result[uid] = moved[uid] + elif dup in moved: + result[uid] = moved[dup] + return Folded(result, frozenset(moved)) + + +def place_line(texts: Iterable[str]) -> str: + """The line a feed row shows for a role's places: the first, and "+N" for + the N others. Repeated texts count once.""" + found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip())) + return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found) diff --git a/app/static/app.css b/app/static/app.css index 9376299..ee87a56 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -942,6 +942,7 @@ body[data-reader="open"] .company-strip { display: none; } line-height: 1.4; } .also a b { display: block; color: var(--faint); font-size: .78rem; font-weight: 600; } +.also span { display: block; margin-bottom: .55rem; color: var(--dim); font-size: .92rem; line-height: 1.4; } @media (hover: hover) { .also a:hover { color: var(--fg); } } diff --git a/app/static/roles.js b/app/static/roles.js index 635c057..92f4d70 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -885,6 +885,42 @@ `; } + /* The role's places in the host's order, one entry for each place text a + link opens: the copies' own links, and the role's link for its own. */ + function placeEntries(r) { + const seen = new Set(); + const own = safeUrl(r.original_url); + const out = []; + for (const p of Array.isArray(r.places) ? r.places : []) { + const url = p.url ? safeUrl(p.url) : own; + const key = `${p.url || ""}\u0001${p.position}\u0001${p.text}`; + if (!p.text || seen.has(key)) continue; + seen.add(key); + out.push({ text: p.text, url }); + } + return out; + } + + /* The place line of the reader: the first place and "+N" for the others, + as the feed's row shows it. */ + function placeLine(r) { + const texts = [...new Set(placeEntries(r).map((e) => e.text))]; + return texts.length > 1 ? `${texts[0]} +${texts.length - 1}` : r.place || texts[0] || ""; + } + + /* Every place of a role that has several, each a link to its own posting, + laid out as "Also posted". */ + function placesList(r) { + const entries = placeEntries(r); + if (new Set(entries.map((e) => e.text)).size < 2) return ""; + return `
`; + } + function drawRow(el, role) { const scored = role.score !== null && role.score !== undefined; const pct = scored ? Math.round(Math.min(1, Math.max(0, role.score)) * 100) : 0; @@ -1243,7 +1279,10 @@ : !loaded ? `Loading…
` : text ? paragraphs(text) - : `The board gave no description for this one.
`; + : (r.teaser || "").trim() + ? `${paragraphs(r.teaser.trim())}` + : `The board gave no description for this one.
`; const pay = payDisplay(r); const careers = safeUrl(companies.get(r.company_key)?.careers_url); const waiting = state.summarising === state.readingUid; @@ -1264,7 +1303,7 @@ ${careers ? `Careers ↗` : ""} - ${r.place ? `${esc(r.place)}` : ""} + ${placeLine(r) ? `${esc(placeLine(r))}` : ""} ${r.first_seen ? `${ago(r.first_seen)}` : ""} ${badges(r)} ${tagChips(r.tags)} @@ -1287,6 +1326,7 @@Read it on ${esc(host(out) || "the board")} ↗
` : ""} + ${loaded ? placesList(r) : ""} ${loaded ? alsoPosted(r) : ""}