Files
jobboard/app/dupes.py
T
WolfandClaude Sonnet 5.5 d060bee5b7 Add the rule that decides which open roles are one posting
Pure module app/dupes.py: towns, same_posting, keep_order and assign, with
tests on the live cross-host pairs, the SAP requisitions and the edge cases.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-09-29 22:06:08 +02:00

180 lines
7.7 KiB
Python

"""Which open roles are one posting, and which copy is kept.
One job is often listed more than once: by a search provider and by the
company's own board, or by one board under several requisition ids (SAP lists
one job under 3 to 9 of them). The stored cluster key `(company_key,
title_key, place_key)` finds none of the cross-host pairs, because the places
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
`Lund, Skåne, Sweden`. So the rule here compares towns, not place strings.
Two rows are one posting when they have the same non-empty `company_key`, the
same `title_key`, the same `country` (or either has none), and the sets of
towns their places name share a member, where a row with no town matches
every town. Host does not matter. The same title in Berlin and in Munich
stays two roles, so a place filter still finds each. The join is transitive:
if A matches B and B matches C, all three are one group, although A and C
alone would not match (a row with no town bridges two towns for this reason).
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
it. Pure by design: no store, no SQL, no network, so the rule is tested on
the strings boards really write and the storage layer only feeds it rows.
"""
from __future__ import annotations
import re
from collections.abc import Iterable
from dataclasses import dataclass
from app.models import _COUNTRY_NAMES, _place_fold
#: Hosts that search across companies rather than run a company's own board.
#: A board closes a role reliably and carries the full text, so it is kept
#: over a copy found here.
SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"})
@dataclass(frozen=True)
class Copy:
"""The columns of one open role the rule reads. `posted_at` is an ISO
string or None; `has_description` is whether `description` is non-empty."""
uid: str
company_key: str
title_key: str
country: str
place: str
host: str
has_description: bool
posted_at: str | None
#: Folded names of every country, in every language `_COUNTRY_NAMES` has.
_COUNTRIES: frozenset[str] = frozenset(
_place_fold(name)
for line in _COUNTRY_NAMES.split("\n") if line
for name in line.split(" ", 1)[1].split("|"))
#: Town names boards spell two ways, folded, to the one spelling used here.
_ALIASES: dict[str, str] = {
"kobenhavn": "copenhagen", "koebenhavn": "copenhagen", "munchen": "munich",
"koln": "cologne", "gothenburg": "goteborg", "wien": "vienna",
"praha": "prague", "warszawa": "warsaw",
}
#: Regions that follow a town and name no town of their own. Dropped only
#: when a town is left, so "Skåne" alone stays a place to compare.
_REGIONS: frozenset[str] = frozenset({
"skane", "scania", "sjaelland", "zealand", "hovedstaden", "capital region",
"midtjylland", "syddanmark", "nordjylland", "jylland", "stockholms lan",
"vastra gotaland", "uppsala lan", "oresund", "bavaria", "bayern",
"baden wurttemberg", "hessen", "nordrhein westfalen", "north rhine westphalia",
"england", "scotland", "wales", "emea", "europe", "eu", "dach", "nordics",
})
#: How the role is worked, not where. Removed from a part, so "Remote Berlin"
#: is Berlin, and a part that is only this is dropped.
_WORK = re.compile(
r"\b(?:hybrid|remote|position|mulighed for hjemmearbejde|hjemmearbejde|"
r"home ?office|work from home|on ?site|onsite|distributed|flexible)\b")
_SPLIT = re.compile(r"[,;|/()\n]")
_AND = re.compile(r" (?:og|och|und|and) ")
#: A postal district or code after a town: "kobenhavn k", "aarhus c", "2100".
_DISTRICT = re.compile(r"(?: (?:[a-z]|\d{2,5}))+$")
def towns(place: str, country: str) -> frozenset[str]:
"""The towns a place line names, folded: "Lund, Skåne, Sweden" is
{lund}, "København K og mulighed for hjemmearbejde" is {copenhagen}, and
"Houston, TX, United States" is {houston}. Empty when it names none.
The line splits at `,` `;` `|` `/` and parentheses, and each part at
` og `, ` och `, ` und ` and ` and `. Each is folded with
`models._place_fold`, then loses work-mode words, a trailing postal
district and its country's ISO code, and is dropped if it is a country
name, a region word or a two-letter US state code beside a town."""
parts: list[tuple[str, str]] = []
for raw in _SPLIT.split(place or ""):
raw = raw.strip()
if not raw:
continue
folded = _place_fold(raw)
pieces = [folded] if folded in _COUNTRIES else _AND.split(f" {folded} ")
parts += [(piece.strip(), raw) for piece in pieces]
found: set[str] = set()
regional: set[str] = set()
for piece, raw in parts:
if not piece or piece in _COUNTRIES or piece == _place_fold(country):
continue
piece = " ".join(_WORK.sub(" ", piece).split())
piece = _DISTRICT.sub("", piece) if " " in piece else piece
if not piece or piece in _COUNTRIES:
continue
if piece in _REGIONS or (len(piece) == 2 and raw.isupper()):
regional.add(piece)
continue
found.add(_ALIASES.get(piece, piece))
return frozenset(found or regional - {_place_fold(country)})
def same_posting(a: Copy, b: Copy) -> bool:
"""Whether two rows are one posting: a non-empty shared `company_key`,
the same `title_key`, compatible countries (equal, or either empty) and
towns that share a member or are empty on either side. Host is ignored."""
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
return False
if a.country and b.country and a.country != b.country:
return False
ta, tb = towns(a.place, a.country), towns(b.place, b.country)
return not ta or not tb or bool(ta & tb)
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
"""The sort key that picks the kept copy of a group, smallest first: a
board over a search provider, then one with a description, then the
earliest `posted_at` (none last), then the smallest `uid`, so the choice
is the same on every run."""
return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at,
c.posted_at or "", c.uid)
def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
"""`uid -> dup_of` for every row: the uid of its group's kept copy, None
for the kept copy itself and for a row with no copies.
Rows are partitioned by `(company_key, title_key)`, a row with no
`company_key` standing alone. Within a partition `same_posting` joins
rows transitively (union-find), and each component points every row but
its `keep_order` minimum at that minimum."""
result: dict[str, str | None] = {}
parts: dict[tuple[str, str], list[Copy]] = {}
for row in rows:
result[row.uid] = None
if row.company_key:
parts.setdefault((row.company_key, row.title_key), []).append(row)
for group in parts.values():
if len(group) < 2:
continue
parent = list(range(len(group)))
def find(i: int, parent: list[int] = parent) -> int:
while parent[i] != i:
parent[i] = parent[parent[i]]
i = parent[i]
return i
for i, a in enumerate(group):
for j in range(i + 1, len(group)):
if find(i) != find(j) and same_posting(a, group[j]):
parent[find(j)] = find(i)
comps: dict[int, list[Copy]] = {}
for i, row in enumerate(group):
comps.setdefault(find(i), []).append(row)
for comp in comps.values():
keeper = min(comp, key=keep_order)
for row in comp:
if row is not keeper:
result[row.uid] = keeper.uid
return result