diff --git a/app/dupes.py b/app/dupes.py new file mode 100644 index 0000000..f08d10a --- /dev/null +++ b/app/dupes.py @@ -0,0 +1,179 @@ +"""Which open roles are one posting, and which copy is kept. + +One job is often listed more than once: by a search provider and by the +company's own board, or by one board under several requisition ids (SAP lists +one job under 3 to 9 of them). The stored cluster key `(company_key, +title_key, place_key)` finds none of the cross-host pairs, because the places +never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against +`Lund, Skåne, Sweden`. So the rule here compares towns, not place strings. + +Two rows are one posting when they have the same non-empty `company_key`, the +same `title_key`, the same `country` (or either has none), and the sets of +towns their places name share a member, where a row with no town matches +every town. Host does not matter. The same title in Berlin and in Munich +stays two roles, so a place filter still finds each. The join is transitive: +if A matches B and B matches C, all three are one group, although A and C +alone would not match (a row with no town bridges two towns for this reason). + +Each group keeps one copy (`keep_order`, smallest wins) and the rest point at +it. Pure by design: no store, no SQL, no network, so the rule is tested on +the strings boards really write and the storage layer only feeds it rows. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass + +from app.models import _COUNTRY_NAMES, _place_fold + +#: Hosts that search across companies rather than run a company's own board. +#: A board closes a role reliably and carries the full text, so it is kept +#: over a copy found here. +SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"}) + + +@dataclass(frozen=True) +class Copy: + """The columns of one open role the rule reads. `posted_at` is an ISO + string or None; `has_description` is whether `description` is non-empty.""" + + uid: str + company_key: str + title_key: str + country: str + place: str + host: str + has_description: bool + posted_at: str | None + + +#: Folded names of every country, in every language `_COUNTRY_NAMES` has. +_COUNTRIES: frozenset[str] = frozenset( + _place_fold(name) + for line in _COUNTRY_NAMES.split("\n") if line + for name in line.split(" ", 1)[1].split("|")) + +#: Town names boards spell two ways, folded, to the one spelling used here. +_ALIASES: dict[str, str] = { + "kobenhavn": "copenhagen", "koebenhavn": "copenhagen", "munchen": "munich", + "koln": "cologne", "gothenburg": "goteborg", "wien": "vienna", + "praha": "prague", "warszawa": "warsaw", +} + +#: Regions that follow a town and name no town of their own. Dropped only +#: when a town is left, so "Skåne" alone stays a place to compare. +_REGIONS: frozenset[str] = frozenset({ + "skane", "scania", "sjaelland", "zealand", "hovedstaden", "capital region", + "midtjylland", "syddanmark", "nordjylland", "jylland", "stockholms lan", + "vastra gotaland", "uppsala lan", "oresund", "bavaria", "bayern", + "baden wurttemberg", "hessen", "nordrhein westfalen", "north rhine westphalia", + "england", "scotland", "wales", "emea", "europe", "eu", "dach", "nordics", +}) + +#: How the role is worked, not where. Removed from a part, so "Remote Berlin" +#: is Berlin, and a part that is only this is dropped. +_WORK = re.compile( + r"\b(?:hybrid|remote|position|mulighed for hjemmearbejde|hjemmearbejde|" + r"home ?office|work from home|on ?site|onsite|distributed|flexible)\b") + +_SPLIT = re.compile(r"[,;|/()\n]") +_AND = re.compile(r" (?:og|och|und|and) ") +#: A postal district or code after a town: "kobenhavn k", "aarhus c", "2100". +_DISTRICT = re.compile(r"(?: (?:[a-z]|\d{2,5}))+$") + + +def towns(place: str, country: str) -> frozenset[str]: + """The towns a place line names, folded: "Lund, Skåne, Sweden" is + {lund}, "København K og mulighed for hjemmearbejde" is {copenhagen}, and + "Houston, TX, United States" is {houston}. Empty when it names none. + + The line splits at `,` `;` `|` `/` and parentheses, and each part at + ` og `, ` och `, ` und ` and ` and `. Each is folded with + `models._place_fold`, then loses work-mode words, a trailing postal + district and its country's ISO code, and is dropped if it is a country + name, a region word or a two-letter US state code beside a town.""" + parts: list[tuple[str, str]] = [] + for raw in _SPLIT.split(place or ""): + raw = raw.strip() + if not raw: + continue + folded = _place_fold(raw) + pieces = [folded] if folded in _COUNTRIES else _AND.split(f" {folded} ") + parts += [(piece.strip(), raw) for piece in pieces] + found: set[str] = set() + regional: set[str] = set() + for piece, raw in parts: + if not piece or piece in _COUNTRIES or piece == _place_fold(country): + continue + piece = " ".join(_WORK.sub(" ", piece).split()) + piece = _DISTRICT.sub("", piece) if " " in piece else piece + if not piece or piece in _COUNTRIES: + continue + if piece in _REGIONS or (len(piece) == 2 and raw.isupper()): + regional.add(piece) + continue + found.add(_ALIASES.get(piece, piece)) + return frozenset(found or regional - {_place_fold(country)}) + + +def same_posting(a: Copy, b: Copy) -> bool: + """Whether two rows are one posting: a non-empty shared `company_key`, + the same `title_key`, compatible countries (equal, or either empty) and + towns that share a member or are empty on either side. Host is ignored.""" + if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key: + return False + if a.country and b.country and a.country != b.country: + return False + ta, tb = towns(a.place, a.country), towns(b.place, b.country) + return not ta or not tb or bool(ta & tb) + + +def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]: + """The sort key that picks the kept copy of a group, smallest first: a + board over a search provider, then one with a description, then the + earliest `posted_at` (none last), then the smallest `uid`, so the choice + is the same on every run.""" + return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at, + c.posted_at or "", c.uid) + + +def assign(rows: Iterable[Copy]) -> dict[str, str | None]: + """`uid -> dup_of` for every row: the uid of its group's kept copy, None + for the kept copy itself and for a row with no copies. + + Rows are partitioned by `(company_key, title_key)`, a row with no + `company_key` standing alone. Within a partition `same_posting` joins + rows transitively (union-find), and each component points every row but + its `keep_order` minimum at that minimum.""" + result: dict[str, str | None] = {} + parts: dict[tuple[str, str], list[Copy]] = {} + for row in rows: + result[row.uid] = None + if row.company_key: + parts.setdefault((row.company_key, row.title_key), []).append(row) + for group in parts.values(): + if len(group) < 2: + continue + parent = list(range(len(group))) + + def find(i: int, parent: list[int] = parent) -> int: + while parent[i] != i: + parent[i] = parent[parent[i]] + i = parent[i] + return i + + for i, a in enumerate(group): + for j in range(i + 1, len(group)): + if find(i) != find(j) and same_posting(a, group[j]): + parent[find(j)] = find(i) + comps: dict[int, list[Copy]] = {} + for i, row in enumerate(group): + comps.setdefault(find(i), []).append(row) + for comp in comps.values(): + keeper = min(comp, key=keep_order) + for row in comp: + if row is not keeper: + result[row.uid] = keeper.uid + return result diff --git a/tests/test_dupes.py b/tests/test_dupes.py new file mode 100644 index 0000000..4073baf --- /dev/null +++ b/tests/test_dupes.py @@ -0,0 +1,158 @@ +"""`app.dupes` against postings as boards really wrote them. Pure: no store, +no network. The live cases are the 2026-09-29 database's.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import pytest # noqa: E402 + +from app.dupes import Copy, assign, keep_order, same_posting, towns # noqa: E402 + + +def cp(uid: str = "u", company: str = "acme", title: str = "engineer", country: str = "DE", + place: str = "", host: str = "greenhouse", desc: bool = True, + posted: str | None = None) -> Copy: + return Copy(uid, company, title, country, place, host, desc, posted) + + +#: `(company_key, title_key, country, host_a, place_a, host_b, place_b, kept host)`. +LIVE = [ + ("ericsson", "entry level 6g developer", "SE", + "platsbanken", "Lund", "eightfold", "Lund, Skåne, Sweden", "eightfold"), + ("gn-group", "machine learning engineer", "DK", + "jobindex", "Ballerup", "workday", "Ballerup, Denmark", "workday"), + ("kmd", "student worker finance", "DK", + "jobindex", "Ballerup, Ballerup, Aalborg, Aalborg", "successfactors", "Ballerup, Denmark", + "successfactors"), + ("trackunit", "machine learning engineer", "DK", + "jobindex", "Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", + "teamtailor", "København, Denmark; Aarhus, Denmark; Kolding, Denmark; Aalborg, Denmark", + "teamtailor"), +] + + +@pytest.mark.parametrize("company,title,country,ha,pa,hb,pb,kept", LIVE) +def test_live_cross_host_pairs(company, title, country, ha, pa, hb, pb, kept): + a = cp("a", company, title, country, pa, ha) + b = cp("b", company, title, country, pb, hb) + assert same_posting(a, b) and same_posting(b, a) + result = assign([a, b]) + assert result == ({"a": "b", "b": None} if kept == hb else {"a": None, "b": "a"}) + + +def test_sap_requisitions(): + place = "Garching bei München (Munich), Germany" + title = "forward deployed chief ai engineering expert" + rows = [ + cp("g3", "sap", title, "DE", place, "successfactors", False, "2026-09-02"), + cp("g1", "sap", title, "DE", place, "successfactors", False, "2026-09-01"), + cp("g2", "sap", title, "DE", place, "successfactors", False, "2026-09-01"), + cp("ny", "sap", title, "US", "New York, United States", "successfactors", True, + "2026-08-01"), + ] + # The earliest posted_at, then the smallest uid, is kept; New York is alone + # although it is the only row with a description. + assert assign(rows) == {"g1": None, "g2": "g1", "g3": "g1", "ny": None} + + +def test_houston_state_code(): + a = cp("a", "sap", "x", "US", "Houston, TX, United States") + b = cp("b", "sap", "x", "US", "Houston, United States") + assert same_posting(a, b) + + +def test_copenhagen_district_and_remote_phrase(): + a = cp("a", "acme", "x", "DK", "København K og mulighed for hjemmearbejde") + b = cp("b", "acme", "x", "DK", "Copenhagen, Denmark") + assert towns(a.place, "DK") == {"copenhagen"} + assert same_posting(a, b) + + +def test_different_towns_are_not_the_same(): + assert not same_posting(cp("a", place="Berlin, Germany"), cp("b", place="Munich, Germany")) + assert assign([cp("a", place="Berlin"), cp("b", place="Munich")]) == {"a": None, "b": None} + + +def test_empty_place_matches(): + assert same_posting(cp("a", place=""), cp("b", place="Berlin")) + assert same_posting(cp("a", place=""), cp("b", place="")) + assert same_posting(cp("a", place="Germany"), cp("b", place="Berlin, Germany")) + + +def test_empty_company_never_grouped(): + a, b = cp("a", company=""), cp("b", company="") + assert not same_posting(a, b) + assert assign([a, b]) == {"a": None, "b": None} + + +def test_other_company_or_title_is_not_the_same(): + assert not same_posting(cp("a"), cp("b", company="other")) + assert not same_posting(cp("a"), cp("b", title="designer")) + + +def test_country_rules(): + assert not same_posting(cp("a", country="DE", place="Berlin"), + cp("b", country="US", place="Berlin")) + assert same_posting(cp("a", country="", place="Berlin"), + cp("b", country="DE", place="Berlin, Germany")) + assert not same_posting(cp("a", country="", place="Berlin"), + cp("b", country="DE", place="Munich")) + + +@pytest.mark.parametrize("place,country,expected", [ + ("Lund, Skåne, Sweden", "SE", {"lund"}), + ("Houston, TX, United States", "US", {"houston"}), + ("New York, United States", "US", {"new york"}), + ("Aarhus C", "DK", {"aarhus"}), + ("Remote Berlin", "DE", {"berlin"}), + ("Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", "DK", + {"aalborg", "copenhagen", "kolding", "aarhus"}), + ("Berlin und Munich", "DE", {"berlin", "munich"}), + ("Germany", "DE", set()), + ("Remote", "", set()), + ("", "", set()), +]) +def test_towns(place, country, expected): + assert towns(place, country) == expected + + +def test_assign_is_transitive(): + # A matches B by Berlin, B matches C by Hamburg, A and C alone do not match. + a = cp("a", place="Berlin", posted="2026-09-03") + b = cp("b", place="Berlin, Hamburg", posted="2026-09-02") + c = cp("c", place="Hamburg", posted="2026-09-01") + assert not same_posting(a, c) + assert assign([a, b, c]) == {"a": "c", "b": "c", "c": None} + + +def test_assign_returns_every_uid(): + rows = [cp("a"), cp("b", title="other"), cp("c", company="")] + assert assign(rows) == {"a": None, "b": None, "c": None} + assert assign([]) == {} + + +def test_assign_separate_groups_in_one_partition(): + rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="Munich"), + cp("d", place="Munich")] + assert assign(rows) == {"a": None, "b": "a", "c": None, "d": "c"} + + +def test_keep_order_ranks(): + board = cp("z", host="workday", desc=False, posted="2026-09-09") + search = cp("a", host="jobindex", desc=True, posted="2026-01-01") + assert min([search, board], key=keep_order) is board + with_desc = cp("z", desc=True, posted="2026-09-09") + without = cp("a", desc=False, posted="2026-01-01") + assert min([without, with_desc], key=keep_order) is with_desc + early = cp("z", posted="2026-01-01") + late = cp("a", posted="2026-02-01") + undated = cp("0", posted=None) + assert sorted([undated, late, early], key=keep_order) == [early, late, undated] + assert min([cp("b", posted="2026-01-01"), cp("a", posted="2026-01-01")], + key=keep_order).uid == "a"