"""Which open roles are one posting, and which copy is kept. One job is often listed more than once: by a search provider and by the company's own board, or by one board under several requisition ids (SAP lists one job under 3 to 9 of them). The stored cluster key `(company_key, title_key, place_key)` finds none of the cross-host pairs, because the places never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against `Lund, Skåne, Sweden`. So the rule here compares towns, not place strings. Two rows are one posting when they have the same non-empty `company_key`, the same `title_key`, the same `country` (or either has none), and the sets of towns their places name share a member, where a row with no town matches every town. Host does not matter. The same title in Berlin and in Munich stays two roles, so a place filter still finds each. The join is transitive: if A matches B and B matches C, all three are one group, although A and C alone would not match (a row with no town bridges two towns for this reason). Each group keeps one copy (`keep_order`, smallest wins) and the rest point at it. Pure by design: no store, no SQL, no network, so the rule is tested on the strings boards really write and the storage layer only feeds it rows. """ from __future__ import annotations import re from collections.abc import Iterable from dataclasses import dataclass from app.models import _COUNTRY_NAMES, _place_fold #: Hosts that search across companies rather than run a company's own board. #: A board closes a role reliably and carries the full text, so it is kept #: over a copy found here. SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"}) @dataclass(frozen=True) class Copy: """The columns of one open role the rule reads. `posted_at` is an ISO string or None; `has_description` is whether `description` is non-empty.""" uid: str company_key: str title_key: str country: str place: str host: str has_description: bool posted_at: str | None #: Folded names of every country, in every language `_COUNTRY_NAMES` has. _COUNTRIES: frozenset[str] = frozenset( _place_fold(name) for line in _COUNTRY_NAMES.split("\n") if line for name in line.split(" ", 1)[1].split("|")) #: Town names boards spell two ways, folded, to the one spelling used here. _ALIASES: dict[str, str] = { "kobenhavn": "copenhagen", "koebenhavn": "copenhagen", "munchen": "munich", "koln": "cologne", "gothenburg": "goteborg", "wien": "vienna", "praha": "prague", "warszawa": "warsaw", } #: Regions that follow a town and name no town of their own. Dropped only #: when a town is left, so "Skåne" alone stays a place to compare. _REGIONS: frozenset[str] = frozenset({ "skane", "scania", "sjaelland", "zealand", "hovedstaden", "capital region", "midtjylland", "syddanmark", "nordjylland", "jylland", "stockholms lan", "vastra gotaland", "uppsala lan", "oresund", "bavaria", "bayern", "baden wurttemberg", "hessen", "nordrhein westfalen", "north rhine westphalia", "england", "scotland", "wales", "emea", "europe", "eu", "dach", "nordics", }) #: How the role is worked, not where. Removed from a part, so "Remote Berlin" #: is Berlin, and a part that is only this is dropped. _WORK = re.compile( r"\b(?:hybrid|remote|position|mulighed for hjemmearbejde|hjemmearbejde|" r"home ?office|work from home|on ?site|onsite|distributed|flexible)\b") _SPLIT = re.compile(r"[,;|/()\n]") _AND = re.compile(r" (?:og|och|und|and) ") #: A postal district or code after a town: "kobenhavn k", "aarhus c", "2100". _DISTRICT = re.compile(r"(?: (?:[a-z]|\d{2,5}))+$") def towns(place: str, country: str) -> frozenset[str]: """The towns a place line names, folded: "Lund, Skåne, Sweden" is {lund}, "København K og mulighed for hjemmearbejde" is {copenhagen}, and "Houston, TX, United States" is {houston}. Empty when it names none. The line splits at `,` `;` `|` `/` and parentheses, and each part at ` og `, ` och `, ` und ` and ` and `. Each is folded with `models._place_fold`, then loses work-mode words, a trailing postal district and its country's ISO code, and is dropped if it is a country name, a region word or a two-letter US state code beside a town.""" parts: list[tuple[str, str]] = [] for raw in _SPLIT.split(place or ""): raw = raw.strip() if not raw: continue folded = _place_fold(raw) pieces = [folded] if folded in _COUNTRIES else _AND.split(f" {folded} ") parts += [(piece.strip(), raw) for piece in pieces] found: set[str] = set() regional: set[str] = set() for piece, raw in parts: if not piece or piece in _COUNTRIES or piece == _place_fold(country): continue piece = " ".join(_WORK.sub(" ", piece).split()) piece = _DISTRICT.sub("", piece) if " " in piece else piece if not piece or piece in _COUNTRIES: continue if piece in _REGIONS or (len(piece) == 2 and raw.isupper()): regional.add(piece) continue found.add(_ALIASES.get(piece, piece)) return frozenset(found or regional - {_place_fold(country)}) def same_posting(a: Copy, b: Copy) -> bool: """Whether two rows are one posting: a non-empty shared `company_key`, the same `title_key`, compatible countries (equal, or either empty) and towns that share a member or are empty on either side. Host is ignored.""" if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key: return False if a.country and b.country and a.country != b.country: return False ta, tb = towns(a.place, a.country), towns(b.place, b.country) return not ta or not tb or bool(ta & tb) def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]: """The sort key that picks the kept copy of a group, smallest first: a board over a search provider, then one with a description, then the earliest `posted_at` (none last), then the smallest `uid`, so the choice is the same on every run.""" return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at, c.posted_at or "", c.uid) def assign(rows: Iterable[Copy]) -> dict[str, str | None]: """`uid -> dup_of` for every row: the uid of its group's kept copy, None for the kept copy itself and for a row with no copies. Rows are partitioned by `(company_key, title_key)`, a row with no `company_key` standing alone. Within a partition `same_posting` joins rows transitively (union-find), and each component points every row but its `keep_order` minimum at that minimum.""" result: dict[str, str | None] = {} parts: dict[tuple[str, str], list[Copy]] = {} for row in rows: result[row.uid] = None if row.company_key: parts.setdefault((row.company_key, row.title_key), []).append(row) for group in parts.values(): if len(group) < 2: continue parent = list(range(len(group))) def find(i: int, parent: list[int] = parent) -> int: while parent[i] != i: parent[i] = parent[parent[i]] i = parent[i] return i for i, a in enumerate(group): for j in range(i + 1, len(group)): if find(i) != find(j) and same_posting(a, group[j]): parent[find(j)] = find(i) comps: dict[int, list[Copy]] = {} for i, row in enumerate(group): comps.setdefault(find(i), []).append(row) for comp in comps.values(): keeper = min(comp, key=keep_order) for row in comp: if row is not keeper: result[row.uid] = keeper.uid return result