Merge branch 'wp-e2-copies'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
695d38c339
6 files changed
+336
-16
No files matched your search
+97
-1
@@ -17,14 +17,30 @@ rows that each match all the others, never a chain: a row with no place
|
|||||||
matches Belgium and the Netherlands, and must not make them one role.
|
matches Belgium and the Netherlands, and must not make them one role.
|
||||||
|
|
||||||
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
|
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
|
||||||
it. Pure by design: no store, no SQL, no network, so the rule is tested on
|
it.
|
||||||
|
|
||||||
|
A second rule folds what the first leaves apart on purpose: one posting per
|
||||||
|
place. A host that lists one job once per office, each with its own id, gives
|
||||||
|
rows with the same company and title whose towns never meet. `fold` merges
|
||||||
|
those rows of one host into the kept role of the first group, when the texts
|
||||||
|
do not say they are different jobs (`similar`), and reports them in
|
||||||
|
`Folded.placed`: the store gives the kept role each such copy's place with the
|
||||||
|
copy's own link. This holds for every host, not only readers that declare
|
||||||
|
`READS.places == "copies"`: on the live database 1,739 company-and-title
|
||||||
|
groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and
|
||||||
|
Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts
|
||||||
|
share fewer than `SAME_TEXT` of their words, or without a link to keep, or
|
||||||
|
listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on
|
||||||
the strings boards really write and the storage layer only feeds it rows.
|
the strings boards really write and the storage layer only feeds it rows.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from functools import lru_cache
|
||||||
|
from typing import NamedTuple
|
||||||
|
|
||||||
from app import places
|
from app import places
|
||||||
from app.models import country_of
|
from app.models import country_of
|
||||||
@@ -48,6 +64,11 @@ class Copy:
|
|||||||
host: str
|
host: str
|
||||||
has_description: bool
|
has_description: bool
|
||||||
posted_at: str | None
|
posted_at: str | None
|
||||||
|
#: The start of the description, else the teaser, for `similar`; '' when
|
||||||
|
#: the row has neither.
|
||||||
|
text: str = ""
|
||||||
|
#: The link that opens this row on its host.
|
||||||
|
url: str = ""
|
||||||
|
|
||||||
|
|
||||||
def _country(c: Copy) -> str:
|
def _country(c: Copy) -> str:
|
||||||
@@ -118,3 +139,78 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
|
|||||||
if row is not keeper:
|
if row is not keeper:
|
||||||
result[row.uid] = keeper.uid
|
result[row.uid] = keeper.uid
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
#: The share of words two texts must have in common (Jaccard, over the first
|
||||||
|
#: 3,000 characters) for one job in two places. Measured on the live
|
||||||
|
#: database: copies of a posting share nearly all of it, two jobs that
|
||||||
|
#: happen to share a title share little more than boilerplate, and a
|
||||||
|
#: translation shares none, which keeps it apart.
|
||||||
|
SAME_TEXT = 0.3
|
||||||
|
|
||||||
|
TEXT_CHARS = 3000
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=8192)
|
||||||
|
def _words(text: str) -> frozenset[str]:
|
||||||
|
return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower()))
|
||||||
|
|
||||||
|
|
||||||
|
def similar(a: str, b: str) -> bool:
|
||||||
|
"""Whether two texts can be one job: a text that is missing says nothing
|
||||||
|
against it, else they share at least `SAME_TEXT` of their words."""
|
||||||
|
wa, wb = _words(a), _words(b)
|
||||||
|
if not wa or not wb:
|
||||||
|
return True
|
||||||
|
return len(wa & wb) / len(wa | wb) >= SAME_TEXT
|
||||||
|
|
||||||
|
|
||||||
|
class Folded(NamedTuple):
|
||||||
|
"""`assign`'s answer plus the rows that are one place of another role:
|
||||||
|
`placed` holds the uid of each copy that `fold` merged by place."""
|
||||||
|
|
||||||
|
dup_of: dict[str, str | None]
|
||||||
|
placed: frozenset[str]
|
||||||
|
|
||||||
|
|
||||||
|
def fold(rows: Iterable[Copy]) -> Folded:
|
||||||
|
"""`assign`, then the per-place copies of each host folded in.
|
||||||
|
|
||||||
|
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
||||||
|
again, in `keep_order`: a row joins the first group whose members are all
|
||||||
|
of its host and `similar` to it. Only a row with a link can be folded, as
|
||||||
|
its link is what the kept role keeps. Every row of a merged group points
|
||||||
|
at its first row, the rows that pointed at a folded one included."""
|
||||||
|
rows = list(rows)
|
||||||
|
result = assign(rows)
|
||||||
|
by_uid = {r.uid: r for r in rows}
|
||||||
|
parts: dict[tuple[str, str], list[Copy]] = {}
|
||||||
|
for uid, dup in result.items():
|
||||||
|
row = by_uid[uid]
|
||||||
|
if dup is None and row.company_key and row.url:
|
||||||
|
parts.setdefault((row.company_key, row.title_key), []).append(row)
|
||||||
|
moved: dict[str, str] = {}
|
||||||
|
for part in parts.values():
|
||||||
|
groups: list[list[Copy]] = []
|
||||||
|
for row in sorted(part, key=keep_order):
|
||||||
|
for group in groups:
|
||||||
|
if all(m.host == row.host and similar(m.text, row.text) for m in group):
|
||||||
|
group.append(row)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
groups.append([row])
|
||||||
|
for keeper, *others in groups:
|
||||||
|
moved.update((other.uid, keeper.uid) for other in others)
|
||||||
|
for uid, dup in result.items():
|
||||||
|
if uid in moved:
|
||||||
|
result[uid] = moved[uid]
|
||||||
|
elif dup in moved:
|
||||||
|
result[uid] = moved[dup]
|
||||||
|
return Folded(result, frozenset(moved))
|
||||||
|
|
||||||
|
|
||||||
|
def place_line(texts: Iterable[str]) -> str:
|
||||||
|
"""The line a feed row shows for a role's places: the first, and "+N" for
|
||||||
|
the N others. Repeated texts count once."""
|
||||||
|
found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip()))
|
||||||
|
return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found)
|
||||||
@@ -942,6 +942,7 @@ body[data-reader="open"] .company-strip { display: none; }
|
|||||||
line-height: 1.4;
|
line-height: 1.4;
|
||||||
}
|
}
|
||||||
.also a b { display: block; color: var(--faint); font-size: .78rem; font-weight: 600; }
|
.also a b { display: block; color: var(--faint); font-size: .78rem; font-weight: 600; }
|
||||||
|
.also span { display: block; margin-bottom: .55rem; color: var(--dim); font-size: .92rem; line-height: 1.4; }
|
||||||
@media (hover: hover) {
|
@media (hover: hover) {
|
||||||
.also a:hover { color: var(--fg); }
|
.also a:hover { color: var(--fg); }
|
||||||
}
|
}
|
||||||
|
|||||||
+42
-2
@@ -885,6 +885,42 @@
|
|||||||
</div>`;
|
</div>`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The role's places in the host's order, one entry for each place text a
|
||||||
|
link opens: the copies' own links, and the role's link for its own. */
|
||||||
|
function placeEntries(r) {
|
||||||
|
const seen = new Set();
|
||||||
|
const own = safeUrl(r.original_url);
|
||||||
|
const out = [];
|
||||||
|
for (const p of Array.isArray(r.places) ? r.places : []) {
|
||||||
|
const url = p.url ? safeUrl(p.url) : own;
|
||||||
|
const key = `${p.url || ""}\u0001${p.position}\u0001${p.text}`;
|
||||||
|
if (!p.text || seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
out.push({ text: p.text, url });
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The place line of the reader: the first place and "+N" for the others,
|
||||||
|
as the feed's row shows it. */
|
||||||
|
function placeLine(r) {
|
||||||
|
const texts = [...new Set(placeEntries(r).map((e) => e.text))];
|
||||||
|
return texts.length > 1 ? `${texts[0]} +${texts.length - 1}` : r.place || texts[0] || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Every place of a role that has several, each a link to its own posting,
|
||||||
|
laid out as "Also posted". */
|
||||||
|
function placesList(r) {
|
||||||
|
const entries = placeEntries(r);
|
||||||
|
if (new Set(entries.map((e) => e.text)).size < 2) return "";
|
||||||
|
return `<div class="also places">
|
||||||
|
<h2 class="reader-label">Places</h2>
|
||||||
|
${entries.map((e) => e.url
|
||||||
|
? `<a href="${esc(e.url)}" target="_blank" rel="noopener">${esc(e.text)} ↗</a>`
|
||||||
|
: `<span>${esc(e.text)}</span>`).join("")}
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
function drawRow(el, role) {
|
function drawRow(el, role) {
|
||||||
const scored = role.score !== null && role.score !== undefined;
|
const scored = role.score !== null && role.score !== undefined;
|
||||||
const pct = scored ? Math.round(Math.min(1, Math.max(0, role.score)) * 100) : 0;
|
const pct = scored ? Math.round(Math.min(1, Math.max(0, role.score)) * 100) : 0;
|
||||||
@@ -1243,7 +1279,10 @@
|
|||||||
: !loaded
|
: !loaded
|
||||||
? `<p class="reader-note">Loading…</p>`
|
? `<p class="reader-note">Loading…</p>`
|
||||||
: text ? paragraphs(text)
|
: text ? paragraphs(text)
|
||||||
: `<p class="reader-note">The board gave no description for this one.</p>`;
|
: (r.teaser || "").trim()
|
||||||
|
? `<p class="reader-note reader-teaser">A short description from the board's
|
||||||
|
list; the board gave no full text.</p>${paragraphs(r.teaser.trim())}`
|
||||||
|
: `<p class="reader-note">The board gave no description for this one.</p>`;
|
||||||
const pay = payDisplay(r);
|
const pay = payDisplay(r);
|
||||||
const careers = safeUrl(companies.get(r.company_key)?.careers_url);
|
const careers = safeUrl(companies.get(r.company_key)?.careers_url);
|
||||||
const waiting = state.summarising === state.readingUid;
|
const waiting = state.summarising === state.readingUid;
|
||||||
@@ -1264,7 +1303,7 @@
|
|||||||
${careers ? `<a class="reader-careers" href="${esc(careers)}" target="_blank" rel="noopener"
|
${careers ? `<a class="reader-careers" href="${esc(careers)}" target="_blank" rel="noopener"
|
||||||
title="The company's careers page">Careers ↗</a>` : ""}
|
title="The company's careers page">Careers ↗</a>` : ""}
|
||||||
<span class="affinity-slot"></span>
|
<span class="affinity-slot"></span>
|
||||||
${r.place ? `<span>${esc(r.place)}</span>` : ""}
|
${placeLine(r) ? `<span>${esc(placeLine(r))}</span>` : ""}
|
||||||
${r.first_seen ? `<span title="First seen">${ago(r.first_seen)}</span>` : ""}
|
${r.first_seen ? `<span title="First seen">${ago(r.first_seen)}</span>` : ""}
|
||||||
${badges(r)}
|
${badges(r)}
|
||||||
${tagChips(r.tags)}
|
${tagChips(r.tags)}
|
||||||
@@ -1287,6 +1326,7 @@
|
|||||||
<div class="prose">${body}</div>
|
<div class="prose">${body}</div>
|
||||||
${out ? `<p class="reader-out"><a href="${esc(out)}" target="_blank" rel="noopener"
|
${out ? `<p class="reader-out"><a href="${esc(out)}" target="_blank" rel="noopener"
|
||||||
>Read it on ${esc(host(out) || "the board")} ↗</a></p>` : ""}
|
>Read it on ${esc(host(out) || "the board")} ↗</a></p>` : ""}
|
||||||
|
${loaded ? placesList(r) : ""}
|
||||||
${loaded ? alsoPosted(r) : ""}
|
${loaded ? alsoPosted(r) : ""}
|
||||||
</div>
|
</div>
|
||||||
<div class="reader-foot">
|
<div class="reader-foot">
|
||||||
|
|||||||
+75
-7
@@ -10,7 +10,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
|
||||||
from contextlib import closing, contextmanager
|
from contextlib import closing, contextmanager
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -287,6 +287,13 @@ def _int_or_none(value: Any) -> int | None:
|
|||||||
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
|
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
|
||||||
|
|
||||||
|
|
||||||
|
#: A copy that is not one place of its kept role: the copies a role lists
|
||||||
|
#: under "Also posted" and counts as "x N". `{kept}` is the kept role's uid
|
||||||
|
#: as SQL, and a place copy is the one whose link the kept role holds.
|
||||||
|
_NOT_A_PLACE = ("original_url NOT IN (SELECT url FROM role_places "
|
||||||
|
"WHERE uid = {kept} AND url != '')")
|
||||||
|
|
||||||
|
|
||||||
class DupeCounts(NamedTuple):
|
class DupeCounts(NamedTuple):
|
||||||
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
|
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
|
||||||
open roles that are a copy, and rows whose `dup_of` it rewrote."""
|
open roles that are a copy, and rows whose `dup_of` it rewrote."""
|
||||||
@@ -296,8 +303,35 @@ class DupeCounts(NamedTuple):
|
|||||||
changed: int
|
changed: int
|
||||||
|
|
||||||
|
|
||||||
|
def _assign_only(rows: Iterable[dupes.Copy]) -> dupes.Folded:
|
||||||
|
return dupes.Folded(dupes.assign(rows), frozenset())
|
||||||
|
|
||||||
|
|
||||||
|
def _sync_copy_places(conn: sqlite3.Connection, placed: dict[str, str | None]) -> None:
|
||||||
|
"""Make the `role_places` rows with a `url` those of the copies `placed`
|
||||||
|
(`copy uid -> kept uid`): each own place of a copy, on the kept role,
|
||||||
|
with the copy's link. Written only when they differ from what is stored,
|
||||||
|
so an unchanged database is not touched. A copy that closed, or a group
|
||||||
|
that dissolved, is no longer in `placed`, and its rows go."""
|
||||||
|
want: set[tuple[str, int, str, str, int | None, str]] = set()
|
||||||
|
if placed:
|
||||||
|
marks = ",".join("?" * len(placed))
|
||||||
|
for uid, pos, text, fold, place_id, url in conn.execute(
|
||||||
|
f"""SELECT rp.uid, rp.position, rp.text, rp.fold, rp.place_id, r.original_url
|
||||||
|
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
||||||
|
WHERE rp.url = '' AND rp.uid IN ({marks})""", list(placed)):
|
||||||
|
want.add((str(placed[uid]), pos, text, fold, place_id, url))
|
||||||
|
have = set(conn.execute(
|
||||||
|
"SELECT uid, position, text, fold, place_id, url FROM role_places WHERE url != ''"))
|
||||||
|
if want != have:
|
||||||
|
conn.execute("DELETE FROM role_places WHERE url != ''")
|
||||||
|
conn.executemany(
|
||||||
|
"INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?, ?)", sorted(want, key=lambda r: (r[0], r[5], r[1], r[2])))
|
||||||
|
|
||||||
|
|
||||||
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
||||||
"""Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside
|
"""Recompute `roles.dup_of` from scratch with `app.dupes.fold`, inside
|
||||||
the caller's transaction: `Store.rebuild_dupes` and migration 18.
|
the caller's transaction: `Store.rebuild_dupes` and migration 18.
|
||||||
|
|
||||||
Only the open roles of a `(company_key, title_key)` with more than one
|
Only the open roles of a `(company_key, title_key)` with more than one
|
||||||
@@ -315,26 +349,40 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
|||||||
them moves to it unless that agent already voted on it; a vote that
|
them moves to it unless that agent already voted on it; a vote that
|
||||||
cannot move stays where it is. An unchanged group is not touched, so
|
cannot move stays where it is. An unchanged group is not touched, so
|
||||||
marking a kept role unread again sticks. Applications keep their
|
marking a kept role unread again sticks. Applications keep their
|
||||||
`role_uid`, and scores of a copy go at the agent's next `score`."""
|
`role_uid`, and scores of a copy go at the agent's next `score`.
|
||||||
|
|
||||||
|
A copy the host published for another place (`dupes.Folded.placed`)
|
||||||
|
leaves its places on the kept role as `role_places` rows with the copy's
|
||||||
|
link, see `_sync_copy_places`."""
|
||||||
|
# Migration 18 runs this on a file that has no `teaser` and no
|
||||||
|
# `role_places` yet: there it groups as before and folds no places.
|
||||||
|
current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)"))
|
||||||
|
text = "COALESCE(NULLIF(description, ''), teaser)" if current else "description"
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
||||||
description != '' AS described, posted_at
|
description != '' AS described, posted_at,
|
||||||
|
SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url
|
||||||
FROM roles
|
FROM roles
|
||||||
WHERE closed_at IS NULL AND company_key != ''
|
WHERE closed_at IS NULL AND company_key != ''
|
||||||
AND (company_key, title_key) IN (
|
AND (company_key, title_key) IN (
|
||||||
SELECT company_key, title_key FROM roles
|
SELECT company_key, title_key FROM roles
|
||||||
WHERE closed_at IS NULL AND company_key != ''
|
WHERE closed_at IS NULL AND company_key != ''
|
||||||
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
|
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
|
||||||
new = dupes.assign(
|
made = dupes.fold if current else _assign_only
|
||||||
|
folded = made(
|
||||||
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
||||||
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None)
|
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None,
|
||||||
|
text=r[8] or "", url=r[9] or "")
|
||||||
for r in rows)
|
for r in rows)
|
||||||
|
new = folded.dup_of
|
||||||
old: dict[str, str | None] = dict.fromkeys(new)
|
old: dict[str, str | None] = dict.fromkeys(new)
|
||||||
old.update((r[0], r[1]) for r in conn.execute(
|
old.update((r[0], r[1]) for r in conn.execute(
|
||||||
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
|
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
|
||||||
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
|
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
|
||||||
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
|
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
|
||||||
|
|
||||||
|
if current:
|
||||||
|
_sync_copy_places(conn, {uid: new[uid] for uid in folded.placed})
|
||||||
changed = {uid for _, uid in updates}
|
changed = {uid for _, uid in updates}
|
||||||
members: dict[str, set[str]] = {}
|
members: dict[str, set[str]] = {}
|
||||||
for uid, dup in new.items():
|
for uid, dup in new.items():
|
||||||
@@ -713,7 +761,8 @@ class Store:
|
|||||||
or a role without any. See `Role.copies`."""
|
or a role without any. See `Role.copies`."""
|
||||||
return [RoleCopy.from_row(r) for r in self._rows(
|
return [RoleCopy.from_row(r) for r in self._rows(
|
||||||
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
|
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
|
||||||
WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))]
|
WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")}
|
||||||
|
ORDER BY posted_at, uid""", (uid, uid))]
|
||||||
|
|
||||||
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
|
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
|
||||||
"""`uid -> open copies` for a page of kept roles, one grouped query
|
"""`uid -> open copies` for a page of kept roles, one grouped query
|
||||||
@@ -726,8 +775,25 @@ class Store:
|
|||||||
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
|
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
|
||||||
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
|
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
|
||||||
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
|
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
|
||||||
|
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
|
||||||
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
||||||
|
|
||||||
|
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
|
||||||
|
"""`uid -> the line the feed shows` for a page of roles that have
|
||||||
|
more than one place (`dupes.place_line`); a role with one place or
|
||||||
|
none is absent and keeps `roles.place`."""
|
||||||
|
if not uids:
|
||||||
|
return {}
|
||||||
|
found: dict[str, list[str]] = {}
|
||||||
|
for r in self._rows(
|
||||||
|
"""SELECT uid, text FROM role_places
|
||||||
|
WHERE uid IN (SELECT value FROM json_each(?))
|
||||||
|
ORDER BY uid, url != '', url, position, rowid""",
|
||||||
|
(json.dumps(list(uids)),)):
|
||||||
|
found.setdefault(str(r["uid"]), []).append(str(r["text"]))
|
||||||
|
return {uid: dupes.place_line(texts) for uid, texts in found.items()
|
||||||
|
if len(set(texts)) > 1}
|
||||||
|
|
||||||
def rebuild_dupes(self) -> DupeCounts:
|
def rebuild_dupes(self) -> DupeCounts:
|
||||||
"""Recompute which open roles are copies of one posting, in one
|
"""Recompute which open roles are copies of one posting, in one
|
||||||
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
|
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
|
||||||
@@ -972,9 +1038,11 @@ class Store:
|
|||||||
found.append(RoleRow.from_row(row))
|
found.append(RoleRow.from_row(row))
|
||||||
by_uid = self._tags_by_uid([f.uid for f in found])
|
by_uid = self._tags_by_uid([f.uid for f in found])
|
||||||
copies = self._copy_counts([f.uid for f in found])
|
copies = self._copy_counts([f.uid for f in found])
|
||||||
|
lines = self._place_lines([f.uid for f in found])
|
||||||
for f in found:
|
for f in found:
|
||||||
f.tags = by_uid.get(f.uid, [])
|
f.tags = by_uid.get(f.uid, [])
|
||||||
f.copies = copies.get(f.uid, 0)
|
f.copies = copies.get(f.uid, 0)
|
||||||
|
f.place = lines.get(f.uid, f.place)
|
||||||
return found
|
return found
|
||||||
|
|
||||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||||
|
|||||||
+48
-1
@@ -12,7 +12,15 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
import pytest # noqa: E402
|
import pytest # noqa: E402
|
||||||
|
|
||||||
from app.dupes import Copy, assign, keep_order, same_posting # noqa: E402
|
from app.dupes import ( # noqa: E402
|
||||||
|
Copy,
|
||||||
|
assign,
|
||||||
|
fold,
|
||||||
|
keep_order,
|
||||||
|
place_line,
|
||||||
|
same_posting,
|
||||||
|
similar,
|
||||||
|
)
|
||||||
from app.places import towns # noqa: E402
|
from app.places import towns # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
@@ -194,3 +202,42 @@ def test_a_row_without_a_place_never_bridges_two_countries():
|
|||||||
assert got == {"a": None, "b": "a", "c": "a", "d": None, "e": None}
|
assert got == {"a": None, "b": "a", "c": "a", "d": None, "e": None}
|
||||||
assert not same_posting(cp("a", country="EU", place="Belgium"),
|
assert not same_posting(cp("a", country="EU", place="Belgium"),
|
||||||
cp("b", country="EU", place="Netherlands"))
|
cp("b", country="EU", place="Netherlands"))
|
||||||
|
|
||||||
|
|
||||||
|
def pc(uid: str, place: str, text: str = "", host: str = "greenhouse", url: str = "u",
|
||||||
|
title: str = "engineer") -> Copy:
|
||||||
|
return Copy(uid, "acme", title, "DE", place, host, bool(text), None, text, url or "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_one_posting_per_place_folds_into_the_first_and_only_within_a_host() -> None:
|
||||||
|
text = "We build the payment platform for our bank customers in Kotlin."
|
||||||
|
got = fold([pc("b", "Berlin", text), pc("m", "Munich", text), pc("h", "Hamburg", text),
|
||||||
|
pc("x", "Kiel", text, host="lever")])
|
||||||
|
assert got.dup_of == {"b": None, "m": "b", "h": "b", "x": None}
|
||||||
|
assert got.placed == {"m", "h"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_folding_needs_the_same_title_a_link_and_texts_that_can_be_one_job() -> None:
|
||||||
|
a = "Build payment services in Kotlin for our bank customers and their teams."
|
||||||
|
b = "Negotiate enterprise cloud contracts with procurement and legal buyers."
|
||||||
|
got = fold([pc("1", "Berlin", a), pc("2", "Munich", b), pc("3", "Hamburg", a, url=""),
|
||||||
|
pc("4", "Kiel", a, title="manager")])
|
||||||
|
assert got.dup_of == {"1": None, "2": None, "3": None, "4": None}
|
||||||
|
assert fold([pc("1", "Berlin", a), pc("2", "Munich", "")]).placed == {"2"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_copy_of_a_folded_row_follows_it_to_the_new_kept_role() -> None:
|
||||||
|
"""Berlin is kept over Munich; a second Munich row that was Munich's copy
|
||||||
|
now points at Berlin, and is not a place of its own."""
|
||||||
|
text = "We build the payment platform for our bank customers in Kotlin."
|
||||||
|
same = fold([pc("b", "Berlin", text), pc("m", "Munich", text),
|
||||||
|
pc("m2", "Munich, Germany", text)])
|
||||||
|
assert same.dup_of == {"b": None, "m": "b", "m2": "b"}
|
||||||
|
assert same.placed == {"m"} # m2 is m's own copy, not another place
|
||||||
|
|
||||||
|
|
||||||
|
def test_similar_and_place_line() -> None:
|
||||||
|
assert similar("", "anything at all") and similar("a job in Kotlin", "a job in Kotlin")
|
||||||
|
assert not similar("sell cloud contracts to buyers", "build kotlin payment services")
|
||||||
|
assert place_line(["Berlin", "Munich", "Berlin", "Hamburg"]) == "Berlin +2"
|
||||||
|
assert place_line(["Berlin", " "]) == "Berlin" and place_line([]) == ""
|
||||||
@@ -37,9 +37,9 @@ def dup_of(store: Store) -> dict[str, str | None]:
|
|||||||
def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart(
|
def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart(
|
||||||
store: Store) -> None:
|
store: Store) -> None:
|
||||||
"""SAP's repeats: three Garching rows are one kept role (the one with
|
"""SAP's repeats: three Garching rows are one kept role (the one with
|
||||||
the description) and two copies; New York is its own role."""
|
the description) and two copies; New York, listed by another host, is its own role."""
|
||||||
garching = [role("1", body=""), role("2"), role("3", body="")]
|
garching = [role("1", body=""), role("2"), role("3", body="")]
|
||||||
york = role("4", place="New York, NY, US")
|
york = role("4", place="New York, NY, US", board="greenhouse")
|
||||||
for item in (*garching, york):
|
for item in (*garching, york):
|
||||||
store.upsert_role(item)
|
store.upsert_role(item)
|
||||||
|
|
||||||
@@ -61,12 +61,80 @@ def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store)
|
|||||||
assert dup_of(store) == {board.uid: None, search.uid: board.uid}
|
assert dup_of(store) == {board.uid: None, search.uid: board.uid}
|
||||||
|
|
||||||
|
|
||||||
def test_berlin_and_munich_stay_two_roles(store: Store) -> None:
|
def kept_of(store: Store, *items: Item) -> str:
|
||||||
for item in (role("b", place="Berlin"), role("m", place="Munich")):
|
"""The uid of the one of these roles that is not a copy."""
|
||||||
|
found = dup_of(store)
|
||||||
|
(kept,) = [i.uid for i in items if found[i.uid] is None]
|
||||||
|
return kept
|
||||||
|
|
||||||
|
|
||||||
|
def test_berlin_and_munich_of_one_host_fold_into_one_role_with_two_places(
|
||||||
|
store: Store) -> None:
|
||||||
|
"""One posting per place on one host is one role that lists both places,
|
||||||
|
each with its own link; the count of copies does not show it as `x2`."""
|
||||||
|
berlin, munich = role("b", place="Berlin"), role("m", place="Munich")
|
||||||
|
for item in (berlin, munich):
|
||||||
|
store.upsert_role(item)
|
||||||
|
assert store.rebuild_dupes() == (1, 1, 1)
|
||||||
|
kept = kept_of(store, berlin, munich)
|
||||||
|
|
||||||
|
got = store.role(kept)
|
||||||
|
assert got is not None
|
||||||
|
assert sorted(p.text for p in got.places) == ["Berlin", "Munich"]
|
||||||
|
assert sorted(p.url for p in got.places if p.url) in (
|
||||||
|
["https://jobs.example/b"], ["https://jobs.example/m"])
|
||||||
|
assert got.copies == []
|
||||||
|
(row,) = store.roles_page()
|
||||||
|
assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None:
|
||||||
|
a = role("a", place="Berlin", body="Build payment services in Kotlin for our bank " * 5)
|
||||||
|
b = role("b", place="Munich", body="Sell cloud contracts to enterprise buyers " * 5)
|
||||||
|
c = role("c", place="Hamburg", board="greenhouse", body=a.body)
|
||||||
|
for item in (a, b, c):
|
||||||
store.upsert_role(item)
|
store.upsert_role(item)
|
||||||
assert store.rebuild_dupes() == (0, 0, 0)
|
assert store.rebuild_dupes() == (0, 0, 0)
|
||||||
|
|
||||||
|
|
||||||
|
def own_and_copy_places(store: Store, uid: str) -> list[tuple[str, bool]]:
|
||||||
|
got = store.role(uid)
|
||||||
|
assert got is not None
|
||||||
|
return sorted((p.text, bool(p.url)) for p in got.places)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_place_copy_closing_or_its_board_off_takes_its_place_away(store: Store) -> None:
|
||||||
|
from app.jobs.sources import set_fetch
|
||||||
|
sub = store.add_subscription("ats", "successfactors:sap", label="SAP")
|
||||||
|
items = [role("b", place="Berlin"), role("m", place="Munich"), role("k", place="Kiel")]
|
||||||
|
for item in items:
|
||||||
|
store.upsert_role(item, sub)
|
||||||
|
store.rebuild_dupes()
|
||||||
|
kept = kept_of(store, *items)
|
||||||
|
assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Kiel", "Munich"]
|
||||||
|
|
||||||
|
closing = next(i for i in items if i.uid != kept)
|
||||||
|
with sqlite3.connect(store.path) as conn:
|
||||||
|
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (closing.uid,))
|
||||||
|
store.rebuild_dupes()
|
||||||
|
assert len(own_and_copy_places(store, kept)) == 2
|
||||||
|
assert store.rebuild_dupes().changed == 0
|
||||||
|
|
||||||
|
set_fetch(store, sub.id, False)
|
||||||
|
assert set(dup_of(store).values()) == {None}
|
||||||
|
assert not any(url for _, url in own_and_copy_places(store, kept))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_repull_keeps_the_copies_places(store: Store) -> None:
|
||||||
|
items = [role("b", place="Berlin"), role("m", place="Munich")]
|
||||||
|
for item in items:
|
||||||
|
store.upsert_role(item)
|
||||||
|
store.rebuild_dupes()
|
||||||
|
kept = kept_of(store, *items)
|
||||||
|
store.upsert_role(next(i for i in items if i.uid == kept))
|
||||||
|
assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Munich"]
|
||||||
|
|
||||||
|
|
||||||
def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared(
|
def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared(
|
||||||
store: Store) -> None:
|
store: Store) -> None:
|
||||||
first, second, third = role("1", day=18), role("2", day=19), role("3", day=20)
|
first, second, third = role("1", day=18), role("2", day=19), role("3", day=20)
|
||||||
@@ -129,7 +197,7 @@ def test_a_closed_kept_role_hands_its_read_and_vote_to_the_next(store: Store) ->
|
|||||||
|
|
||||||
|
|
||||||
def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None:
|
def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None:
|
||||||
kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin")
|
kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin", board="greenhouse")
|
||||||
for item in (kept, copy, other):
|
for item in (kept, copy, other):
|
||||||
store.upsert_role(item)
|
store.upsert_role(item)
|
||||||
store.set_tags(item.uid, ["python"])
|
store.set_tags(item.uid, ["python"])
|
||||||
|
|||||||
Reference in new issue
Block a user