Files
jobboard/app/dupes.py
T
WolfandClaude Sonnet 5.5 6e6e655203 Wire freehire, Jobnet and Akademikernes Jobbank into the CLI, the Sources form, dupes and the docs
Provider names, the CLI's choices and the dupes rule's search hosts now come from readers.PROVIDERS, and the regions, work modes and remote words from the modules' REGIONS, MODES and REMOTE, read with getattr so main works before and after the readers merge.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-03 12:24:34 +02:00

230 lines
9.6 KiB
Python

"""Which open roles are one posting, and which copy is kept.
One job is often listed more than once: by a search provider and by the
company's own board, or by one board under several requisition ids (SAP lists
one job under 3 to 9 of them). The stored cluster key `(company_key,
title_key, place_key)` finds none of the cross-host pairs, because the places
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
`Lund, Skåne, Sweden`. So the rule here compares towns as `places.towns` reads
them, not place strings.
Two rows are one posting when they have the same non-empty `company_key`, the
same `title_key`, the same `country` (or either has none), and the sets of
towns their places name share a member, where a row with no town matches
every town. Host does not matter. The same title in Berlin and in Munich
stays two roles, so a place filter still finds each. A group is a set of
rows that each match all the others, never a chain: a row with no place
matches Belgium and the Netherlands, and must not make them one role.
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
it.
A second rule folds what the first leaves apart on purpose: one posting per
place. A host that lists one job once per office, each with its own id, gives
rows with the same company and title whose towns never meet. `fold` merges
those rows of one host into the kept role of the first group, when the texts
do not say they are different jobs (`similar`), and reports them in
`Folded.placed`: the store gives the kept role each such copy's place with the
copy's own link. This holds for every host, not only readers that declare
`READS.places == "copies"`: on the live database 1,739 company-and-title
groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and
Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts
share fewer than `SAME_TEXT` of their words, or without a link to keep, or
listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on
the strings boards really write and the storage layer only feeds it rows.
"""
from __future__ import annotations
import re
import sys
from collections.abc import Iterable
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import NamedTuple
from app import places
from app.models import country_of
def is_search_host(host: str) -> bool:
"""Whether `host` is a provider that searches across companies rather than
a company's own board: a reader of `scope="search"`. A board closes a role
reliably and carries the full text, so it is kept over a copy found here.
`plugins` is a top-level folder beside `app`, reached lazily, with the
repo root put on the path when it is not there yet, as
`app.models._declared_posted` does."""
try:
from plugins import readers
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import readers
return host in readers.PROVIDERS
@dataclass(frozen=True)
class Copy:
"""The columns of one open role the rule reads. `posted_at` is an ISO
string or None; `has_description` is whether `description` is non-empty."""
uid: str
company_key: str
title_key: str
country: str
place: str
host: str
has_description: bool
posted_at: str | None
#: The start of the description, else the teaser, for `similar`; '' when
#: the row has neither.
text: str = ""
#: The link that opens this row on its host.
url: str = ""
def _country(c: Copy) -> str:
"""The row's country, or the one its place names when the column is empty
or `EU`, a board's word for several offices: "Netherlands" is NL."""
return c.country if c.country not in ("", "EU") else country_of(c.place)
def _tier(c: Copy) -> int:
"""How precisely a row says where: 0 names a town, 1 only a country, 2
neither."""
return 0 if places.towns(c.place) else 1 if _country(c) else 2
def same_posting(a: Copy, b: Copy) -> bool:
"""Whether two rows are one posting: a non-empty shared `company_key`,
the same `title_key`, compatible countries (equal, or either empty) and
towns that share a member or are empty on either side. Host is ignored."""
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
return False
ca, cb = _country(a), _country(b)
if ca and cb and ca != cb:
return False
ta, tb = places.towns(a.place), places.towns(b.place)
return not ta or not tb or bool(ta & tb)
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
"""The sort key that picks the kept copy of a group, smallest first: a
board over a search provider, then one with a description, then the
earliest `posted_at` (none last), then the smallest `uid`, so the choice
is the same on every run."""
return (is_search_host(c.host), not c.has_description, not c.posted_at,
c.posted_at or "", c.uid)
def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
"""`uid -> dup_of` for every row: the uid of its group's kept copy, None
for the kept copy itself and for a row with no copies.
Rows are partitioned by `(company_key, title_key)`, a row with no
`company_key` standing alone. Within a partition rows are taken sharpest
tier first and in `keep_order`, and a row joins a group only when it is `same_posting` with every member.
Joining on one match would chain: a row with no country matches Japan and
Malaysia both and would make them one. A row naming a town that fits two
groups joins the first. A vaguer row joins the first group of its own tier
it fits, else the one sharper group it fits, else starts its own: with no
place it fits Belgium and the Netherlands, and joins neither. Each group keeps its `keep_order`
minimum, so a board with no place still wins over a search with a town."""
result: dict[str, str | None] = {}
parts: dict[tuple[str, str], list[Copy]] = {}
for row in rows:
result[row.uid] = None
if row.company_key:
parts.setdefault((row.company_key, row.title_key), []).append(row)
for part in parts.values():
groups: list[list[Copy]] = []
for row in sorted(part, key=lambda r: (_tier(r), keep_order(r))):
fits = [g for g in groups if all(same_posting(row, m) for m in g)]
level = [g for g in fits if _tier(g[0]) == _tier(row)]
if level or len(fits) == 1:
(level or fits)[0].append(row)
else:
groups.append([row])
for group in groups:
keeper = min(group, key=keep_order)
for row in group:
if row is not keeper:
result[row.uid] = keeper.uid
return result
#: The share of words two texts must have in common (Jaccard, over the first
#: 3,000 characters) for one job in two places. Measured on the live
#: database: copies of a posting share nearly all of it, two jobs that
#: happen to share a title share little more than boilerplate, and a
#: translation shares none, which keeps it apart.
SAME_TEXT = 0.3
TEXT_CHARS = 3000
@lru_cache(maxsize=8192)
def _words(text: str) -> frozenset[str]:
return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower()))
def similar(a: str, b: str) -> bool:
"""Whether two texts can be one job: a text that is missing says nothing
against it, else they share at least `SAME_TEXT` of their words."""
wa, wb = _words(a), _words(b)
if not wa or not wb:
return True
return len(wa & wb) / len(wa | wb) >= SAME_TEXT
class Folded(NamedTuple):
"""`assign`'s answer plus the rows that are one place of another role:
`placed` holds the uid of each copy that `fold` merged by place."""
dup_of: dict[str, str | None]
placed: frozenset[str]
def fold(rows: Iterable[Copy]) -> Folded:
"""`assign`, then the per-place copies of each host folded in.
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
again, in `keep_order`: a row joins the first group whose members are all
of its host and `similar` to it. Only a row with a link can be folded, as
its link is what the kept role keeps. Every row of a merged group points
at its first row, the rows that pointed at a folded one included."""
rows = list(rows)
result = assign(rows)
by_uid = {r.uid: r for r in rows}
parts: dict[tuple[str, str], list[Copy]] = {}
for uid, dup in result.items():
row = by_uid[uid]
if dup is None and row.company_key and row.url:
parts.setdefault((row.company_key, row.title_key), []).append(row)
moved: dict[str, str] = {}
for part in parts.values():
groups: list[list[Copy]] = []
for row in sorted(part, key=keep_order):
for group in groups:
if all(m.host == row.host and similar(m.text, row.text) for m in group):
group.append(row)
break
else:
groups.append([row])
for keeper, *others in groups:
moved.update((other.uid, keeper.uid) for other in others)
for uid, dup in result.items():
if uid in moved:
result[uid] = moved[uid]
elif dup in moved:
result[uid] = moved[dup]
return Folded(result, frozenset(moved))
def place_line(texts: Iterable[str]) -> str:
"""The line a feed row shows for a role's places: the first, and "+N" for
the N others. Repeated texts count once."""
found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip()))
return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found)