Fold per-place copies of one host into one role that lists every place with its own link
The reader shows every place, each linked to its copy, and the teaser marked as a short description when there is no text. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3adafd8950
commit
3d1fed7c0e
6 files changed
+336
-16
No files matched your search
+97
-1
@@ -17,14 +17,30 @@ rows that each match all the others, never a chain: a row with no place
|
||||
matches Belgium and the Netherlands, and must not make them one role.
|
||||
|
||||
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
|
||||
it. Pure by design: no store, no SQL, no network, so the rule is tested on
|
||||
it.
|
||||
|
||||
A second rule folds what the first leaves apart on purpose: one posting per
|
||||
place. A host that lists one job once per office, each with its own id, gives
|
||||
rows with the same company and title whose towns never meet. `fold` merges
|
||||
those rows of one host into the kept role of the first group, when the texts
|
||||
do not say they are different jobs (`similar`), and reports them in
|
||||
`Folded.placed`: the store gives the kept role each such copy's place with the
|
||||
copy's own link. This holds for every host, not only readers that declare
|
||||
`READS.places == "copies"`: on the live database 1,739 company-and-title
|
||||
groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and
|
||||
Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts
|
||||
share fewer than `SAME_TEXT` of their words, or without a link to keep, or
|
||||
listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on
|
||||
the strings boards really write and the storage layer only feeds it rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import NamedTuple
|
||||
|
||||
from app import places
|
||||
from app.models import country_of
|
||||
@@ -48,6 +64,11 @@ class Copy:
|
||||
host: str
|
||||
has_description: bool
|
||||
posted_at: str | None
|
||||
#: The start of the description, else the teaser, for `similar`; '' when
|
||||
#: the row has neither.
|
||||
text: str = ""
|
||||
#: The link that opens this row on its host.
|
||||
url: str = ""
|
||||
|
||||
|
||||
def _country(c: Copy) -> str:
|
||||
@@ -118,3 +139,78 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
|
||||
if row is not keeper:
|
||||
result[row.uid] = keeper.uid
|
||||
return result
|
||||
|
||||
|
||||
#: The share of words two texts must have in common (Jaccard, over the first
|
||||
#: 3,000 characters) for one job in two places. Measured on the live
|
||||
#: database: copies of a posting share nearly all of it, two jobs that
|
||||
#: happen to share a title share little more than boilerplate, and a
|
||||
#: translation shares none, which keeps it apart.
|
||||
SAME_TEXT = 0.3
|
||||
|
||||
TEXT_CHARS = 3000
|
||||
|
||||
|
||||
@lru_cache(maxsize=8192)
|
||||
def _words(text: str) -> frozenset[str]:
|
||||
return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower()))
|
||||
|
||||
|
||||
def similar(a: str, b: str) -> bool:
|
||||
"""Whether two texts can be one job: a text that is missing says nothing
|
||||
against it, else they share at least `SAME_TEXT` of their words."""
|
||||
wa, wb = _words(a), _words(b)
|
||||
if not wa or not wb:
|
||||
return True
|
||||
return len(wa & wb) / len(wa | wb) >= SAME_TEXT
|
||||
|
||||
|
||||
class Folded(NamedTuple):
|
||||
"""`assign`'s answer plus the rows that are one place of another role:
|
||||
`placed` holds the uid of each copy that `fold` merged by place."""
|
||||
|
||||
dup_of: dict[str, str | None]
|
||||
placed: frozenset[str]
|
||||
|
||||
|
||||
def fold(rows: Iterable[Copy]) -> Folded:
|
||||
"""`assign`, then the per-place copies of each host folded in.
|
||||
|
||||
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
||||
again, in `keep_order`: a row joins the first group whose members are all
|
||||
of its host and `similar` to it. Only a row with a link can be folded, as
|
||||
its link is what the kept role keeps. Every row of a merged group points
|
||||
at its first row, the rows that pointed at a folded one included."""
|
||||
rows = list(rows)
|
||||
result = assign(rows)
|
||||
by_uid = {r.uid: r for r in rows}
|
||||
parts: dict[tuple[str, str], list[Copy]] = {}
|
||||
for uid, dup in result.items():
|
||||
row = by_uid[uid]
|
||||
if dup is None and row.company_key and row.url:
|
||||
parts.setdefault((row.company_key, row.title_key), []).append(row)
|
||||
moved: dict[str, str] = {}
|
||||
for part in parts.values():
|
||||
groups: list[list[Copy]] = []
|
||||
for row in sorted(part, key=keep_order):
|
||||
for group in groups:
|
||||
if all(m.host == row.host and similar(m.text, row.text) for m in group):
|
||||
group.append(row)
|
||||
break
|
||||
else:
|
||||
groups.append([row])
|
||||
for keeper, *others in groups:
|
||||
moved.update((other.uid, keeper.uid) for other in others)
|
||||
for uid, dup in result.items():
|
||||
if uid in moved:
|
||||
result[uid] = moved[uid]
|
||||
elif dup in moved:
|
||||
result[uid] = moved[dup]
|
||||
return Folded(result, frozenset(moved))
|
||||
|
||||
|
||||
def place_line(texts: Iterable[str]) -> str:
|
||||
"""The line a feed row shows for a role's places: the first, and "+N" for
|
||||
the N others. Repeated texts count once."""
|
||||
found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip()))
|
||||
return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found)
|
||||
Reference in new issue
Block a user