Merge branch 'wp-h-dupes'

This commit is contained in:
Wolf committed 2026-09-29 22:12:39 +02:00
commit 93bdad72a2
5 files changed
+335 -14

No files matched your search

+120
View File
@@ -0,0 +1,120 @@
"""Which open roles are one posting, and which copy is kept.
One job is often listed more than once: by a search provider and by the
company's own board, or by one board under several requisition ids (SAP lists
one job under 3 to 9 of them). The stored cluster key `(company_key,
title_key, place_key)` finds none of the cross-host pairs, because the places
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
`Lund, Skåne, Sweden`. So the rule here compares towns as `places.towns` reads
them, not place strings.
Two rows are one posting when they have the same non-empty `company_key`, the
same `title_key`, the same `country` (or either has none), and the sets of
towns their places name share a member, where a row with no town matches
every town. Host does not matter. The same title in Berlin and in Munich
stays two roles, so a place filter still finds each. A group is a set of
rows that each match all the others, never a chain: a row with no place
matches Belgium and the Netherlands, and must not make them one role.
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
it. Pure by design: no store, no SQL, no network, so the rule is tested on
the strings boards really write and the storage layer only feeds it rows.
"""
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
from app import places
from app.models import country_of
#: Hosts that search across companies rather than run a company's own board.
#: A board closes a role reliably and carries the full text, so it is kept
#: over a copy found here.
SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"})
@dataclass(frozen=True)
class Copy:
"""The columns of one open role the rule reads. `posted_at` is an ISO
string or None; `has_description` is whether `description` is non-empty."""
uid: str
company_key: str
title_key: str
country: str
place: str
host: str
has_description: bool
posted_at: str | None
def _country(c: Copy) -> str:
"""The row's country, or the one its place names when the column is empty
or `EU`, a board's word for several offices: "Netherlands" is NL."""
return c.country if c.country not in ("", "EU") else country_of(c.place)
def _tier(c: Copy) -> int:
"""How precisely a row says where: 0 names a town, 1 only a country, 2
neither."""
return 0 if places.towns(c.place) else 1 if _country(c) else 2
def same_posting(a: Copy, b: Copy) -> bool:
"""Whether two rows are one posting: a non-empty shared `company_key`,
the same `title_key`, compatible countries (equal, or either empty) and
towns that share a member or are empty on either side. Host is ignored."""
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
return False
ca, cb = _country(a), _country(b)
if ca and cb and ca != cb:
return False
ta, tb = places.towns(a.place), places.towns(b.place)
return not ta or not tb or bool(ta & tb)
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
"""The sort key that picks the kept copy of a group, smallest first: a
board over a search provider, then one with a description, then the
earliest `posted_at` (none last), then the smallest `uid`, so the choice
is the same on every run."""
return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at,
c.posted_at or "", c.uid)
def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
"""`uid -> dup_of` for every row: the uid of its group's kept copy, None
for the kept copy itself and for a row with no copies.
Rows are partitioned by `(company_key, title_key)`, a row with no
`company_key` standing alone. Within a partition rows are taken sharpest
tier first and in `keep_order`, and a row joins a group only when it is `same_posting` with every member.
Joining on one match would chain: a row with no country matches Japan and
Malaysia both and would make them one. A row naming a town that fits two
groups joins the first. A vaguer row joins the first group of its own tier
it fits, else the one sharper group it fits, else starts its own: with no
place it fits Belgium and the Netherlands, and joins neither. Each group keeps its `keep_order`
minimum, so a board with no place still wins over a search with a town."""
result: dict[str, str | None] = {}
parts: dict[tuple[str, str], list[Copy]] = {}
for row in rows:
result[row.uid] = None
if row.company_key:
parts.setdefault((row.company_key, row.title_key), []).append(row)
for part in parts.values():
groups: list[list[Copy]] = []
for row in sorted(part, key=lambda r: (_tier(r), keep_order(r))):
fits = [g for g in groups if all(same_posting(row, m) for m in g)]
level = [g for g in fits if _tier(g[0]) == _tier(row)]
if level or len(fits) == 1:
(level or fits)[0].append(row)
else:
groups.append([row])
for group in groups:
keeper = min(group, key=keep_order)
for row in group:
if row is not keeper:
result[row.uid] = keeper.uid
return result
+14 -1
View File
@@ -35,7 +35,7 @@ from functools import lru_cache
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
"label", "match_sql", "matches", "parse", "register", "suggest"]
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
def fold(text: str) -> str:
@@ -578,6 +578,19 @@ def _towns_in(folded: str) -> list[str]:
return list(dict.fromkeys(found))
def towns(place: str) -> frozenset[str]:
"""The towns a line names, each as its group's key, so `København K` and
`Copenhagen, Denmark` are both `{"copenhagen"}`: the first comma part of each
place, and every known town the line holds as whole words, so a list such as
`Aalborg, Copenhagen, Kolding` gives all three. A state or province after
the town is not a town, so `Mountain View, California` and `San Francisco,
California` share none. Empty for a line that names only a country, a region
or how the role is worked."""
found = {_group(fold(town)) for town in _towns(place)}
found.update(_towns_in(fold(place or "")))
return frozenset(found)
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
"""Open roles per region, over folded lines. A line is cut into every run of
words once, and each region asks whether it holds one of its phrases."""