"""Which open roles are one posting, and which copy is kept. One job is often listed more than once: by a search provider and by the company's own board, or by one board under several requisition ids (SAP lists one job under 3 to 9 of them). The stored cluster key `(company_key, title_key, place_key)` finds none of the cross-host pairs, because the places never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against `Lund, Skåne, Sweden`. So the rule here compares towns as `places.towns` reads them, not place strings. Two rows are one posting when they have the same non-empty `company_key`, the same `title_key`, the same `country` (or either has none), and the sets of towns their places name share a member, where a row with no town matches every town. Host does not matter. The same title in Berlin and in Munich stays two roles, so a place filter still finds each. A group is a set of rows that each match all the others, never a chain: a row with no place matches Belgium and the Netherlands, and must not make them one role. Each group keeps one copy (`keep_order`, smallest wins) and the rest point at it. A second rule folds what the first leaves apart on purpose: one posting per place. A host that lists one job once per office, each with its own id, gives rows with the same company and title whose towns never meet. `fold` merges those rows of one host into the kept role of the first group, when the texts do not say they are different jobs (`similar`), and reports them in `Folded.placed`: the store gives the kept role each such copy's place with the copy's own link. This holds for every host, not only readers that declare `READS.places == "copies"`: on the live database 1,739 company-and-title groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts share fewer than `SAME_TEXT` of their words, or without a link to keep, or listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on the strings boards really write and the storage layer only feeds it rows. """ from __future__ import annotations import re import sys from collections.abc import Iterable from dataclasses import dataclass from functools import lru_cache from pathlib import Path from typing import NamedTuple from app import places from app.models import country_of def is_search_host(host: str) -> bool: """Whether `host` is a provider that searches across companies rather than a company's own board: a reader of `scope="search"`. A board closes a role reliably and carries the full text, so it is kept over a copy found here. `plugins` is a top-level folder beside `app`, reached lazily, with the repo root put on the path when it is not there yet, as `app.models._declared_posted` does.""" try: from plugins import readers except ImportError: sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from plugins import readers return host in readers.PROVIDERS @dataclass(frozen=True) class Copy: """The columns of one open role the rule reads. `posted_at` is an ISO string or None; `has_description` is whether `description` is non-empty.""" uid: str company_key: str title_key: str country: str place: str host: str has_description: bool posted_at: str | None #: The start of the description, else the teaser, for `similar`; '' when #: the row has neither. text: str = "" #: The link that opens this row on its host. url: str = "" #: Whether the row is a remote role (`roles.remote` is 'remote'). remote: bool = False def _country(c: Copy) -> str: """The row's country, or the one its place names when the column is empty or `EU`, a board's word for several offices: "Netherlands" is NL.""" return c.country if c.country not in ("", "EU") else country_of(c.place) def _tier(c: Copy) -> int: """How precisely a row says where: 0 names a town, 1 only a country, 2 neither.""" return 0 if places.towns(c.place) else 1 if _country(c) else 2 def same_posting(a: Copy, b: Copy) -> bool: """Whether two rows are one posting: a non-empty shared `company_key`, the same `title_key`, compatible countries (equal, or either empty) and towns that share a member or are empty on either side. Host is ignored.""" if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key: return False ca, cb = _country(a), _country(b) if ca and cb and ca != cb: return False ta, tb = places.towns(a.place), places.towns(b.place) return not ta or not tb or bool(ta & tb) def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]: """The sort key that picks the kept copy of a group, smallest first: a board over a search provider, then one with a description, then the earliest `posted_at` (none last), then the smallest `uid`, so the choice is the same on every run.""" return (is_search_host(c.host), not c.has_description, not c.posted_at, c.posted_at or "", c.uid) def assign(rows: Iterable[Copy]) -> dict[str, str | None]: """`uid -> dup_of` for every row: the uid of its group's kept copy, None for the kept copy itself and for a row with no copies. Rows are partitioned by `(company_key, title_key)`, a row with no `company_key` standing alone. Within a partition rows are taken sharpest tier first and in `keep_order`, and a row joins a group only when it is `same_posting` with every member. Joining on one match would chain: a row with no country matches Japan and Malaysia both and would make them one. A row naming a town that fits two groups joins the first. A vaguer row joins the first group of its own tier it fits, else the one sharper group it fits, else starts its own: with no place it fits Belgium and the Netherlands, and joins neither. Each group keeps its `keep_order` minimum, so a board with no place still wins over a search with a town.""" result: dict[str, str | None] = {} parts: dict[tuple[str, str], list[Copy]] = {} for row in rows: result[row.uid] = None if row.company_key: parts.setdefault((row.company_key, row.title_key), []).append(row) for part in parts.values(): groups: list[list[Copy]] = [] for row in sorted(part, key=lambda r: (_tier(r), keep_order(r))): fits = [g for g in groups if all(same_posting(row, m) for m in g)] level = [g for g in fits if _tier(g[0]) == _tier(row)] if level or len(fits) == 1: (level or fits)[0].append(row) else: groups.append([row]) for group in groups: keeper = min(group, key=keep_order) for row in group: if row is not keeper: result[row.uid] = keeper.uid return result #: The share of words two texts must have in common (Jaccard, over the first #: 3,000 characters) for one job in two places. Measured on the live #: database: copies of a posting share nearly all of it, two jobs that #: happen to share a title share little more than boilerplate, and a #: translation shares none, which keeps it apart. SAME_TEXT = 0.3 TEXT_CHARS = 3000 @lru_cache(maxsize=8192) def _words(text: str) -> frozenset[str]: return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower())) def similar(a: str, b: str) -> bool: """Whether two texts can be one job: a text that is missing says nothing against it, else they share at least `SAME_TEXT` of their words.""" wa, wb = _words(a), _words(b) if not wa or not wb: return True return len(wa & wb) / len(wa | wb) >= SAME_TEXT class Folded(NamedTuple): """`assign`'s answer plus the rows that are one place of another role: `placed` holds the uid of each copy that `fold` merged by place.""" dup_of: dict[str, str | None] placed: frozenset[str] def fold(rows: Iterable[Copy]) -> Folded: """`assign`, then the per-place copies of each host folded in. Within a `(company_key, title_key)` the kept rows of `assign` are grouped again, in `keep_order`: a row joins the first group whose members are all of its host, `similar` to it and remote as it is or not. Only a row with a link can be folded, as its link is what the kept role keeps. Every row of a merged group points at its first row, the rows that pointed at a folded one included. A remote listing and an office listing stay two roles: the kept role takes a copy's places and not its remote, so the fetch gate, which reads a remote role by its area, would reject the remote listing with an office role elsewhere, and the office one with a remote role elsewhere.""" rows = list(rows) result = assign(rows) by_uid = {r.uid: r for r in rows} parts: dict[tuple[str, str], list[Copy]] = {} for uid, dup in result.items(): row = by_uid[uid] if dup is None and row.company_key and row.url: parts.setdefault((row.company_key, row.title_key), []).append(row) moved: dict[str, str] = {} for part in parts.values(): groups: list[list[Copy]] = [] for row in sorted(part, key=keep_order): for group in groups: if all(m.host == row.host and m.remote == row.remote and similar(m.text, row.text) for m in group): group.append(row) break else: groups.append([row]) for keeper, *others in groups: moved.update((other.uid, keeper.uid) for other in others) for uid, dup in result.items(): if uid in moved: result[uid] = moved[uid] elif dup in moved: result[uid] = moved[dup] return Folded(result, frozenset(moved)) def place_line(texts: Iterable[str]) -> str: """The line a feed row shows for a role's places: the first, and "+N" for the N others. Repeated texts count once.""" found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip())) return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found)