From d060bee5b705076ad1c73489de02c7fdd8741b3d Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 22:06:08 +0200 Subject: [PATCH 1/3] Add the rule that decides which open roles are one posting Pure module app/dupes.py: towns, same_posting, keep_order and assign, with tests on the live cross-host pairs, the SAP requisitions and the edge cases. Co-Authored-By: Claude Sonnet 5.5 --- app/dupes.py | 179 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_dupes.py | 158 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 337 insertions(+) create mode 100644 app/dupes.py create mode 100644 tests/test_dupes.py diff --git a/app/dupes.py b/app/dupes.py new file mode 100644 index 0000000..f08d10a --- /dev/null +++ b/app/dupes.py @@ -0,0 +1,179 @@ +"""Which open roles are one posting, and which copy is kept. + +One job is often listed more than once: by a search provider and by the +company's own board, or by one board under several requisition ids (SAP lists +one job under 3 to 9 of them). The stored cluster key `(company_key, +title_key, place_key)` finds none of the cross-host pairs, because the places +never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against +`Lund, Skåne, Sweden`. So the rule here compares towns, not place strings. + +Two rows are one posting when they have the same non-empty `company_key`, the +same `title_key`, the same `country` (or either has none), and the sets of +towns their places name share a member, where a row with no town matches +every town. Host does not matter. The same title in Berlin and in Munich +stays two roles, so a place filter still finds each. The join is transitive: +if A matches B and B matches C, all three are one group, although A and C +alone would not match (a row with no town bridges two towns for this reason). + +Each group keeps one copy (`keep_order`, smallest wins) and the rest point at +it. Pure by design: no store, no SQL, no network, so the rule is tested on +the strings boards really write and the storage layer only feeds it rows. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass + +from app.models import _COUNTRY_NAMES, _place_fold + +#: Hosts that search across companies rather than run a company's own board. +#: A board closes a role reliably and carries the full text, so it is kept +#: over a copy found here. +SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"}) + + +@dataclass(frozen=True) +class Copy: + """The columns of one open role the rule reads. `posted_at` is an ISO + string or None; `has_description` is whether `description` is non-empty.""" + + uid: str + company_key: str + title_key: str + country: str + place: str + host: str + has_description: bool + posted_at: str | None + + +#: Folded names of every country, in every language `_COUNTRY_NAMES` has. +_COUNTRIES: frozenset[str] = frozenset( + _place_fold(name) + for line in _COUNTRY_NAMES.split("\n") if line + for name in line.split(" ", 1)[1].split("|")) + +#: Town names boards spell two ways, folded, to the one spelling used here. +_ALIASES: dict[str, str] = { + "kobenhavn": "copenhagen", "koebenhavn": "copenhagen", "munchen": "munich", + "koln": "cologne", "gothenburg": "goteborg", "wien": "vienna", + "praha": "prague", "warszawa": "warsaw", +} + +#: Regions that follow a town and name no town of their own. Dropped only +#: when a town is left, so "Skåne" alone stays a place to compare. +_REGIONS: frozenset[str] = frozenset({ + "skane", "scania", "sjaelland", "zealand", "hovedstaden", "capital region", + "midtjylland", "syddanmark", "nordjylland", "jylland", "stockholms lan", + "vastra gotaland", "uppsala lan", "oresund", "bavaria", "bayern", + "baden wurttemberg", "hessen", "nordrhein westfalen", "north rhine westphalia", + "england", "scotland", "wales", "emea", "europe", "eu", "dach", "nordics", +}) + +#: How the role is worked, not where. Removed from a part, so "Remote Berlin" +#: is Berlin, and a part that is only this is dropped. +_WORK = re.compile( + r"\b(?:hybrid|remote|position|mulighed for hjemmearbejde|hjemmearbejde|" + r"home ?office|work from home|on ?site|onsite|distributed|flexible)\b") + +_SPLIT = re.compile(r"[,;|/()\n]") +_AND = re.compile(r" (?:og|och|und|and) ") +#: A postal district or code after a town: "kobenhavn k", "aarhus c", "2100". +_DISTRICT = re.compile(r"(?: (?:[a-z]|\d{2,5}))+$") + + +def towns(place: str, country: str) -> frozenset[str]: + """The towns a place line names, folded: "Lund, Skåne, Sweden" is + {lund}, "København K og mulighed for hjemmearbejde" is {copenhagen}, and + "Houston, TX, United States" is {houston}. Empty when it names none. + + The line splits at `,` `;` `|` `/` and parentheses, and each part at + ` og `, ` och `, ` und ` and ` and `. Each is folded with + `models._place_fold`, then loses work-mode words, a trailing postal + district and its country's ISO code, and is dropped if it is a country + name, a region word or a two-letter US state code beside a town.""" + parts: list[tuple[str, str]] = [] + for raw in _SPLIT.split(place or ""): + raw = raw.strip() + if not raw: + continue + folded = _place_fold(raw) + pieces = [folded] if folded in _COUNTRIES else _AND.split(f" {folded} ") + parts += [(piece.strip(), raw) for piece in pieces] + found: set[str] = set() + regional: set[str] = set() + for piece, raw in parts: + if not piece or piece in _COUNTRIES or piece == _place_fold(country): + continue + piece = " ".join(_WORK.sub(" ", piece).split()) + piece = _DISTRICT.sub("", piece) if " " in piece else piece + if not piece or piece in _COUNTRIES: + continue + if piece in _REGIONS or (len(piece) == 2 and raw.isupper()): + regional.add(piece) + continue + found.add(_ALIASES.get(piece, piece)) + return frozenset(found or regional - {_place_fold(country)}) + + +def same_posting(a: Copy, b: Copy) -> bool: + """Whether two rows are one posting: a non-empty shared `company_key`, + the same `title_key`, compatible countries (equal, or either empty) and + towns that share a member or are empty on either side. Host is ignored.""" + if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key: + return False + if a.country and b.country and a.country != b.country: + return False + ta, tb = towns(a.place, a.country), towns(b.place, b.country) + return not ta or not tb or bool(ta & tb) + + +def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]: + """The sort key that picks the kept copy of a group, smallest first: a + board over a search provider, then one with a description, then the + earliest `posted_at` (none last), then the smallest `uid`, so the choice + is the same on every run.""" + return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at, + c.posted_at or "", c.uid) + + +def assign(rows: Iterable[Copy]) -> dict[str, str | None]: + """`uid -> dup_of` for every row: the uid of its group's kept copy, None + for the kept copy itself and for a row with no copies. + + Rows are partitioned by `(company_key, title_key)`, a row with no + `company_key` standing alone. Within a partition `same_posting` joins + rows transitively (union-find), and each component points every row but + its `keep_order` minimum at that minimum.""" + result: dict[str, str | None] = {} + parts: dict[tuple[str, str], list[Copy]] = {} + for row in rows: + result[row.uid] = None + if row.company_key: + parts.setdefault((row.company_key, row.title_key), []).append(row) + for group in parts.values(): + if len(group) < 2: + continue + parent = list(range(len(group))) + + def find(i: int, parent: list[int] = parent) -> int: + while parent[i] != i: + parent[i] = parent[parent[i]] + i = parent[i] + return i + + for i, a in enumerate(group): + for j in range(i + 1, len(group)): + if find(i) != find(j) and same_posting(a, group[j]): + parent[find(j)] = find(i) + comps: dict[int, list[Copy]] = {} + for i, row in enumerate(group): + comps.setdefault(find(i), []).append(row) + for comp in comps.values(): + keeper = min(comp, key=keep_order) + for row in comp: + if row is not keeper: + result[row.uid] = keeper.uid + return result diff --git a/tests/test_dupes.py b/tests/test_dupes.py new file mode 100644 index 0000000..4073baf --- /dev/null +++ b/tests/test_dupes.py @@ -0,0 +1,158 @@ +"""`app.dupes` against postings as boards really wrote them. Pure: no store, +no network. The live cases are the 2026-09-29 database's.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import pytest # noqa: E402 + +from app.dupes import Copy, assign, keep_order, same_posting, towns # noqa: E402 + + +def cp(uid: str = "u", company: str = "acme", title: str = "engineer", country: str = "DE", + place: str = "", host: str = "greenhouse", desc: bool = True, + posted: str | None = None) -> Copy: + return Copy(uid, company, title, country, place, host, desc, posted) + + +#: `(company_key, title_key, country, host_a, place_a, host_b, place_b, kept host)`. +LIVE = [ + ("ericsson", "entry level 6g developer", "SE", + "platsbanken", "Lund", "eightfold", "Lund, Skåne, Sweden", "eightfold"), + ("gn-group", "machine learning engineer", "DK", + "jobindex", "Ballerup", "workday", "Ballerup, Denmark", "workday"), + ("kmd", "student worker finance", "DK", + "jobindex", "Ballerup, Ballerup, Aalborg, Aalborg", "successfactors", "Ballerup, Denmark", + "successfactors"), + ("trackunit", "machine learning engineer", "DK", + "jobindex", "Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", + "teamtailor", "København, Denmark; Aarhus, Denmark; Kolding, Denmark; Aalborg, Denmark", + "teamtailor"), +] + + +@pytest.mark.parametrize("company,title,country,ha,pa,hb,pb,kept", LIVE) +def test_live_cross_host_pairs(company, title, country, ha, pa, hb, pb, kept): + a = cp("a", company, title, country, pa, ha) + b = cp("b", company, title, country, pb, hb) + assert same_posting(a, b) and same_posting(b, a) + result = assign([a, b]) + assert result == ({"a": "b", "b": None} if kept == hb else {"a": None, "b": "a"}) + + +def test_sap_requisitions(): + place = "Garching bei München (Munich), Germany" + title = "forward deployed chief ai engineering expert" + rows = [ + cp("g3", "sap", title, "DE", place, "successfactors", False, "2026-09-02"), + cp("g1", "sap", title, "DE", place, "successfactors", False, "2026-09-01"), + cp("g2", "sap", title, "DE", place, "successfactors", False, "2026-09-01"), + cp("ny", "sap", title, "US", "New York, United States", "successfactors", True, + "2026-08-01"), + ] + # The earliest posted_at, then the smallest uid, is kept; New York is alone + # although it is the only row with a description. + assert assign(rows) == {"g1": None, "g2": "g1", "g3": "g1", "ny": None} + + +def test_houston_state_code(): + a = cp("a", "sap", "x", "US", "Houston, TX, United States") + b = cp("b", "sap", "x", "US", "Houston, United States") + assert same_posting(a, b) + + +def test_copenhagen_district_and_remote_phrase(): + a = cp("a", "acme", "x", "DK", "København K og mulighed for hjemmearbejde") + b = cp("b", "acme", "x", "DK", "Copenhagen, Denmark") + assert towns(a.place, "DK") == {"copenhagen"} + assert same_posting(a, b) + + +def test_different_towns_are_not_the_same(): + assert not same_posting(cp("a", place="Berlin, Germany"), cp("b", place="Munich, Germany")) + assert assign([cp("a", place="Berlin"), cp("b", place="Munich")]) == {"a": None, "b": None} + + +def test_empty_place_matches(): + assert same_posting(cp("a", place=""), cp("b", place="Berlin")) + assert same_posting(cp("a", place=""), cp("b", place="")) + assert same_posting(cp("a", place="Germany"), cp("b", place="Berlin, Germany")) + + +def test_empty_company_never_grouped(): + a, b = cp("a", company=""), cp("b", company="") + assert not same_posting(a, b) + assert assign([a, b]) == {"a": None, "b": None} + + +def test_other_company_or_title_is_not_the_same(): + assert not same_posting(cp("a"), cp("b", company="other")) + assert not same_posting(cp("a"), cp("b", title="designer")) + + +def test_country_rules(): + assert not same_posting(cp("a", country="DE", place="Berlin"), + cp("b", country="US", place="Berlin")) + assert same_posting(cp("a", country="", place="Berlin"), + cp("b", country="DE", place="Berlin, Germany")) + assert not same_posting(cp("a", country="", place="Berlin"), + cp("b", country="DE", place="Munich")) + + +@pytest.mark.parametrize("place,country,expected", [ + ("Lund, Skåne, Sweden", "SE", {"lund"}), + ("Houston, TX, United States", "US", {"houston"}), + ("New York, United States", "US", {"new york"}), + ("Aarhus C", "DK", {"aarhus"}), + ("Remote Berlin", "DE", {"berlin"}), + ("Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", "DK", + {"aalborg", "copenhagen", "kolding", "aarhus"}), + ("Berlin und Munich", "DE", {"berlin", "munich"}), + ("Germany", "DE", set()), + ("Remote", "", set()), + ("", "", set()), +]) +def test_towns(place, country, expected): + assert towns(place, country) == expected + + +def test_assign_is_transitive(): + # A matches B by Berlin, B matches C by Hamburg, A and C alone do not match. + a = cp("a", place="Berlin", posted="2026-09-03") + b = cp("b", place="Berlin, Hamburg", posted="2026-09-02") + c = cp("c", place="Hamburg", posted="2026-09-01") + assert not same_posting(a, c) + assert assign([a, b, c]) == {"a": "c", "b": "c", "c": None} + + +def test_assign_returns_every_uid(): + rows = [cp("a"), cp("b", title="other"), cp("c", company="")] + assert assign(rows) == {"a": None, "b": None, "c": None} + assert assign([]) == {} + + +def test_assign_separate_groups_in_one_partition(): + rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="Munich"), + cp("d", place="Munich")] + assert assign(rows) == {"a": None, "b": "a", "c": None, "d": "c"} + + +def test_keep_order_ranks(): + board = cp("z", host="workday", desc=False, posted="2026-09-09") + search = cp("a", host="jobindex", desc=True, posted="2026-01-01") + assert min([search, board], key=keep_order) is board + with_desc = cp("z", desc=True, posted="2026-09-09") + without = cp("a", desc=False, posted="2026-01-01") + assert min([without, with_desc], key=keep_order) is with_desc + early = cp("z", posted="2026-01-01") + late = cp("a", posted="2026-02-01") + undated = cp("0", posted=None) + assert sorted([undated, late, early], key=keep_order) == [early, late, undated] + assert min([cp("b", posted="2026-01-01"), cp("a", posted="2026-01-01")], + key=keep_order).uid == "a" From 1e22537ae500132ddf73022786aba082e4adbd9c Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 22:07:05 +0200 Subject: [PATCH 2/3] Let a role that names no town join one group, never bridge two, and place dupes above places Co-Authored-By: Claude Opus 5.5 --- app/dupes.py | 32 +++++++++++++++++++++++++------- tests/test_architecture.py | 2 +- tests/test_dupes.py | 10 ++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/app/dupes.py b/app/dupes.py index f08d10a..9fdeb9b 100644 --- a/app/dupes.py +++ b/app/dupes.py @@ -11,9 +11,10 @@ Two rows are one posting when they have the same non-empty `company_key`, the same `title_key`, the same `country` (or either has none), and the sets of towns their places name share a member, where a row with no town matches every town. Host does not matter. The same title in Berlin and in Munich -stays two roles, so a place filter still finds each. The join is transitive: -if A matches B and B matches C, all three are one group, although A and C -alone would not match (a row with no town bridges two towns for this reason). +stays two roles, so a place filter still finds each. The join is transitive +among rows that name a town. A row that names none joins a group only when +exactly one group of its company and title would take it, so a bare +"Germany" never merges Berlin and Munich into one role. Each group keeps one copy (`keep_order`, smallest wins) and the rest point at it. Pure by design: no store, no SQL, no network, so the rule is tested on @@ -144,9 +145,10 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]: for the kept copy itself and for a row with no copies. Rows are partitioned by `(company_key, title_key)`, a row with no - `company_key` standing alone. Within a partition `same_posting` joins - rows transitively (union-find), and each component points every row but - its `keep_order` minimum at that minimum.""" + `company_key` standing alone. Within a partition `same_posting` joins the + rows that name a town transitively (union-find); a row naming none joins + the one group it matches, or stays out when it matches two. Each + component points every row but its `keep_order` minimum at that minimum.""" result: dict[str, str | None] = {} parts: dict[tuple[str, str], list[Copy]] = {} for row in rows: @@ -164,10 +166,26 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]: i = parent[i] return i + placed = [bool(towns(row.place, row.country)) for row in group] for i, a in enumerate(group): for j in range(i + 1, len(group)): - if find(i) != find(j) and same_posting(a, group[j]): + if (placed[i] and placed[j] and find(i) != find(j) + and same_posting(a, group[j])): parent[find(j)] = find(i) + # A row without a town matches every town, so it would bridge two + # groups it matches; it joins only when there is one to join. Rows + # without a town that fit no placed group group among themselves. + loose = [i for i in range(len(group)) if not placed[i]] + for i in loose: + roots = {find(j) for j in range(len(group)) + if placed[j] and same_posting(group[i], group[j])} + if len(roots) == 1: + parent[i] = roots.pop() + for n, i in enumerate(loose): + for j in loose[n + 1:]: + if (parent[i] == i and parent[j] == j + and same_posting(group[i], group[j])): + parent[j] = i comps: dict[int, list[Copy]] = {} for i, row in enumerate(group): comps.setdefault(find(i), []).append(row) diff --git a/tests/test_architecture.py b/tests/test_architecture.py index 78f0b37..c3dc326 100644 --- a/tests/test_architecture.py +++ b/tests/test_architecture.py @@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1] #: no database, so storage may use them. `main` assembles the app from #: everything below it; `cli` and `__main__` are the entry points and may #: reach anything. -LAYERS = ["models", "places", "seniority", "config", "storage", "jobs", "web", "main", "cli", +LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli", "__main__"] APP = "app" EXTERNAL = {"desk", "pipeline", "auth"} diff --git a/tests/test_dupes.py b/tests/test_dupes.py index 4073baf..82f379c 100644 --- a/tests/test_dupes.py +++ b/tests/test_dupes.py @@ -156,3 +156,13 @@ def test_keep_order_ranks(): assert sorted([undated, late, early], key=keep_order) == [early, late, undated] assert min([cp("b", posted="2026-01-01"), cp("a", posted="2026-01-01")], key=keep_order).uid == "a" + + +def test_a_row_without_a_town_joins_only_one_group(): + # "Germany" matches Berlin and Munich; joining both would make them one role. + rows = [cp("a", place="Berlin"), cp("b", place="Munich"), cp("c", place="Germany")] + assert assign(rows) == {"a": None, "b": None, "c": None} + rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="")] + assert assign(rows) == {"a": None, "b": "a", "c": "a"} + rows = [cp("a", place=""), cp("b", place="Germany")] + assert assign(rows) == {"a": None, "b": "a"} From b09a2f6b33a91e78a5cf33e66219711eb8aa9d11 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 22:12:39 +0200 Subject: [PATCH 3/3] Group copies only when they all match, compare towns through places.towns, and record the rule in the plan Co-Authored-By: Claude Opus 5.5 --- app/dupes.py | 153 ++++++++++------------------------------ app/places.py | 15 +++- docs/plan-duplicates.md | 16 ++--- tests/test_dupes.py | 40 +++++++++-- 4 files changed, 90 insertions(+), 134 deletions(-) diff --git a/app/dupes.py b/app/dupes.py index 9fdeb9b..5c5cc31 100644 --- a/app/dupes.py +++ b/app/dupes.py @@ -5,16 +5,16 @@ company's own board, or by one board under several requisition ids (SAP lists one job under 3 to 9 of them). The stored cluster key `(company_key, title_key, place_key)` finds none of the cross-host pairs, because the places never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against -`Lund, Skåne, Sweden`. So the rule here compares towns, not place strings. +`Lund, Skåne, Sweden`. So the rule here compares towns as `places.towns` reads +them, not place strings. Two rows are one posting when they have the same non-empty `company_key`, the same `title_key`, the same `country` (or either has none), and the sets of towns their places name share a member, where a row with no town matches every town. Host does not matter. The same title in Berlin and in Munich -stays two roles, so a place filter still finds each. The join is transitive -among rows that name a town. A row that names none joins a group only when -exactly one group of its company and title would take it, so a bare -"Germany" never merges Berlin and Munich into one role. +stays two roles, so a place filter still finds each. A group is a set of +rows that each match all the others, never a chain: a row with no place +matches Belgium and the Netherlands, and must not make them one role. Each group keeps one copy (`keep_order`, smallest wins) and the rest point at it. Pure by design: no store, no SQL, no network, so the rule is tested on @@ -23,11 +23,11 @@ the strings boards really write and the storage layer only feeds it rows. from __future__ import annotations -import re from collections.abc import Iterable from dataclasses import dataclass -from app.models import _COUNTRY_NAMES, _place_fold +from app import places +from app.models import country_of #: Hosts that search across companies rather than run a company's own board. #: A board closes a role reliably and carries the full text, so it is kept @@ -50,73 +50,16 @@ class Copy: posted_at: str | None -#: Folded names of every country, in every language `_COUNTRY_NAMES` has. -_COUNTRIES: frozenset[str] = frozenset( - _place_fold(name) - for line in _COUNTRY_NAMES.split("\n") if line - for name in line.split(" ", 1)[1].split("|")) - -#: Town names boards spell two ways, folded, to the one spelling used here. -_ALIASES: dict[str, str] = { - "kobenhavn": "copenhagen", "koebenhavn": "copenhagen", "munchen": "munich", - "koln": "cologne", "gothenburg": "goteborg", "wien": "vienna", - "praha": "prague", "warszawa": "warsaw", -} - -#: Regions that follow a town and name no town of their own. Dropped only -#: when a town is left, so "Skåne" alone stays a place to compare. -_REGIONS: frozenset[str] = frozenset({ - "skane", "scania", "sjaelland", "zealand", "hovedstaden", "capital region", - "midtjylland", "syddanmark", "nordjylland", "jylland", "stockholms lan", - "vastra gotaland", "uppsala lan", "oresund", "bavaria", "bayern", - "baden wurttemberg", "hessen", "nordrhein westfalen", "north rhine westphalia", - "england", "scotland", "wales", "emea", "europe", "eu", "dach", "nordics", -}) - -#: How the role is worked, not where. Removed from a part, so "Remote Berlin" -#: is Berlin, and a part that is only this is dropped. -_WORK = re.compile( - r"\b(?:hybrid|remote|position|mulighed for hjemmearbejde|hjemmearbejde|" - r"home ?office|work from home|on ?site|onsite|distributed|flexible)\b") - -_SPLIT = re.compile(r"[,;|/()\n]") -_AND = re.compile(r" (?:og|och|und|and) ") -#: A postal district or code after a town: "kobenhavn k", "aarhus c", "2100". -_DISTRICT = re.compile(r"(?: (?:[a-z]|\d{2,5}))+$") +def _country(c: Copy) -> str: + """The row's country, or the one its place names when the column is empty + or `EU`, a board's word for several offices: "Netherlands" is NL.""" + return c.country if c.country not in ("", "EU") else country_of(c.place) -def towns(place: str, country: str) -> frozenset[str]: - """The towns a place line names, folded: "Lund, Skåne, Sweden" is - {lund}, "København K og mulighed for hjemmearbejde" is {copenhagen}, and - "Houston, TX, United States" is {houston}. Empty when it names none. - - The line splits at `,` `;` `|` `/` and parentheses, and each part at - ` og `, ` och `, ` und ` and ` and `. Each is folded with - `models._place_fold`, then loses work-mode words, a trailing postal - district and its country's ISO code, and is dropped if it is a country - name, a region word or a two-letter US state code beside a town.""" - parts: list[tuple[str, str]] = [] - for raw in _SPLIT.split(place or ""): - raw = raw.strip() - if not raw: - continue - folded = _place_fold(raw) - pieces = [folded] if folded in _COUNTRIES else _AND.split(f" {folded} ") - parts += [(piece.strip(), raw) for piece in pieces] - found: set[str] = set() - regional: set[str] = set() - for piece, raw in parts: - if not piece or piece in _COUNTRIES or piece == _place_fold(country): - continue - piece = " ".join(_WORK.sub(" ", piece).split()) - piece = _DISTRICT.sub("", piece) if " " in piece else piece - if not piece or piece in _COUNTRIES: - continue - if piece in _REGIONS or (len(piece) == 2 and raw.isupper()): - regional.add(piece) - continue - found.add(_ALIASES.get(piece, piece)) - return frozenset(found or regional - {_place_fold(country)}) +def _tier(c: Copy) -> int: + """How precisely a row says where: 0 names a town, 1 only a country, 2 + neither.""" + return 0 if places.towns(c.place) else 1 if _country(c) else 2 def same_posting(a: Copy, b: Copy) -> bool: @@ -125,9 +68,10 @@ def same_posting(a: Copy, b: Copy) -> bool: towns that share a member or are empty on either side. Host is ignored.""" if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key: return False - if a.country and b.country and a.country != b.country: + ca, cb = _country(a), _country(b) + if ca and cb and ca != cb: return False - ta, tb = towns(a.place, a.country), towns(b.place, b.country) + ta, tb = places.towns(a.place), places.towns(b.place) return not ta or not tb or bool(ta & tb) @@ -145,53 +89,32 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]: for the kept copy itself and for a row with no copies. Rows are partitioned by `(company_key, title_key)`, a row with no - `company_key` standing alone. Within a partition `same_posting` joins the - rows that name a town transitively (union-find); a row naming none joins - the one group it matches, or stays out when it matches two. Each - component points every row but its `keep_order` minimum at that minimum.""" + `company_key` standing alone. Within a partition rows are taken sharpest + tier first and in `keep_order`, and a row joins a group only when it is `same_posting` with every member. + Joining on one match would chain: a row with no country matches Japan and + Malaysia both and would make them one. A row naming a town that fits two + groups joins the first. A vaguer row joins the first group of its own tier + it fits, else the one sharper group it fits, else starts its own: with no + place it fits Belgium and the Netherlands, and joins neither. Each group keeps its `keep_order` + minimum, so a board with no place still wins over a search with a town.""" result: dict[str, str | None] = {} parts: dict[tuple[str, str], list[Copy]] = {} for row in rows: result[row.uid] = None if row.company_key: parts.setdefault((row.company_key, row.title_key), []).append(row) - for group in parts.values(): - if len(group) < 2: - continue - parent = list(range(len(group))) - - def find(i: int, parent: list[int] = parent) -> int: - while parent[i] != i: - parent[i] = parent[parent[i]] - i = parent[i] - return i - - placed = [bool(towns(row.place, row.country)) for row in group] - for i, a in enumerate(group): - for j in range(i + 1, len(group)): - if (placed[i] and placed[j] and find(i) != find(j) - and same_posting(a, group[j])): - parent[find(j)] = find(i) - # A row without a town matches every town, so it would bridge two - # groups it matches; it joins only when there is one to join. Rows - # without a town that fit no placed group group among themselves. - loose = [i for i in range(len(group)) if not placed[i]] - for i in loose: - roots = {find(j) for j in range(len(group)) - if placed[j] and same_posting(group[i], group[j])} - if len(roots) == 1: - parent[i] = roots.pop() - for n, i in enumerate(loose): - for j in loose[n + 1:]: - if (parent[i] == i and parent[j] == j - and same_posting(group[i], group[j])): - parent[j] = i - comps: dict[int, list[Copy]] = {} - for i, row in enumerate(group): - comps.setdefault(find(i), []).append(row) - for comp in comps.values(): - keeper = min(comp, key=keep_order) - for row in comp: + for part in parts.values(): + groups: list[list[Copy]] = [] + for row in sorted(part, key=lambda r: (_tier(r), keep_order(r))): + fits = [g for g in groups if all(same_posting(row, m) for m in g)] + level = [g for g in fits if _tier(g[0]) == _tier(row)] + if level or len(fits) == 1: + (level or fits)[0].append(row) + else: + groups.append([row]) + for group in groups: + keeper = min(group, key=keep_order) + for row in group: if row is not keeper: result[row.uid] = keeper.uid return result diff --git a/app/places.py b/app/places.py index 0505189..005619c 100644 --- a/app/places.py +++ b/app/places.py @@ -35,7 +35,7 @@ from functools import lru_cache from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold __all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known", - "label", "match_sql", "matches", "parse", "register", "suggest"] + "label", "match_sql", "matches", "parse", "register", "suggest", "towns"] def fold(text: str) -> str: @@ -578,6 +578,19 @@ def _towns_in(folded: str) -> list[str]: return list(dict.fromkeys(found)) +def towns(place: str) -> frozenset[str]: + """The towns a line names, each as its group's key, so `København K` and + `Copenhagen, Denmark` are both `{"copenhagen"}`: the first comma part of each + place, and every known town the line holds as whole words, so a list such as + `Aalborg, Copenhagen, Kolding` gives all three. A state or province after + the town is not a town, so `Mountain View, California` and `San Francisco, + California` share none. Empty for a line that names only a country, a region + or how the role is worked.""" + found = {_group(fold(town)) for town in _towns(place)} + found.update(_towns_in(fold(place or ""))) + return frozenset(found) + + def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]: """Open roles per region, over folded lines. A line is cut into every run of words once, and each region asks whether it holds one of its phrases.""" diff --git a/docs/plan-duplicates.md b/docs/plan-duplicates.md index 2db39f1..281e705 100644 --- a/docs/plan-duplicates.md +++ b/docs/plan-duplicates.md @@ -17,7 +17,9 @@ The same-host groups are the larger problem. SAP lists one job under 3 to 9 requ ## Decisions -**Two open roles are one posting when company, title, country and a town agree.** Same non-empty `company_key`, same `title_key`, same `country` (or either empty), and the sets of towns their `place` names share at least one member, where either side having no town counts as a match. Host does not matter, so a board's own repeats collapse like an aggregator's copy. The same title in Berlin and in Munich stays two roles, so a place filter still finds each. +**Two open roles are one posting when company, title, country and a town agree.** Same non-empty `company_key`, same `title_key`, the same country (the column, or the one the place names when the column is empty or `EU`), and town sets from `places.towns` that share a member, where either side having no town counts as a match. `places.towns` takes the first comma part of each place and every known town, so `Mountain View, California` and `San Francisco, California` share none. Host does not matter, so a board's own repeats collapse like an aggregator's copy. The same title in Berlin and in Munich stays two roles, so a place filter still finds each. + +**A group is a set of rows that all match each other, never a chain.** A row with no place matches Belgium and the Netherlands, and a row with no country matches Japan and Malaysia; joining on one match merged both pairs on the live data. Rows are taken sharpest first (a town, then only a country, then neither) and join a group only when they match every member. A vaguer row joins a group of its own tier, else the one sharper group it fits, else starts its own. **The group is stored, not computed per query.** `roles.dup_of` holds the uid of the group's kept role, NULL on the kept role and on a role without copies. The feed, the counts and the facets add `dup_of IS NULL`, which keeps `idx_roles_posted` usable; a window function over the filtered rows would scan them all on every page. @@ -43,17 +45,7 @@ The storage and UI packages share files with packages E and G of `plan-feed-filt |---|---|---| | H, the rule | Sonnet | `app/dupes.py`, `tests/test_dupes.py` | -`app/dupes.py` is pure, with no store and no SQL: - -```python -SEARCH_HOSTS: frozenset[str] # jobindex, adzuna, platsbanken -def towns(place: str, country: str) -> frozenset[str] -def same_posting(a: Copy, b: Copy) -> bool -def keep_order(c: Copy) -> tuple # sort key, smallest is kept -def assign(rows: Iterable[Copy]) -> dict[str, str | None] # uid -> dup_of -``` - -`Copy` is a frozen dataclass with `uid`, `company_key`, `title_key`, `country`, `place`, `host`, `has_description`, `posted_at`. `towns` splits on `,`, `;`, `|`, `/`, ` og `, ` och `, ` und `, ` and `, folds with `models._place_fold`, and drops country names (`models._COUNTRY_NAMES`), `Skåne`-style region words that follow a town, and words such as `hybrid`, `remote`, `position`, `mulighed for hjemmearbejde`; `København` and `Copenhagen` are one town. `assign` partitions rows by `(company_key, title_key)`, joins rows by `same_posting` transitively, and within each component points every row but the `keep_order` minimum at that minimum. The tests carry the five live cases above verbatim, Berlin against Munich, an empty place, an empty company and an empty country. +Done: `app/dupes.py` (`SEARCH_HOSTS`, `Copy`, `same_posting`, `keep_order`, `assign`) over `places.towns`, and `tests/test_dupes.py` with the live cases. On the live database it hides 1,080 of 33,881 open roles in 848 groups, 14 of them across hosts, in half a second. Known leftovers: Zeiss field technicians in `Remote - USA NH` and `Remote - USA WA` collapse, since a state code is not a town. ### Wave 2, after G of `plan-feed-filters.md` is merged, in parallel diff --git a/tests/test_dupes.py b/tests/test_dupes.py index 82f379c..f3f6c22 100644 --- a/tests/test_dupes.py +++ b/tests/test_dupes.py @@ -12,7 +12,8 @@ if str(ROOT) not in sys.path: import pytest # noqa: E402 -from app.dupes import Copy, assign, keep_order, same_posting, towns # noqa: E402 +from app.dupes import Copy, assign, keep_order, same_posting # noqa: E402 +from app.places import towns # noqa: E402 def cp(uid: str = "u", company: str = "acme", title: str = "engineer", country: str = "DE", @@ -70,7 +71,7 @@ def test_houston_state_code(): def test_copenhagen_district_and_remote_phrase(): a = cp("a", "acme", "x", "DK", "København K og mulighed for hjemmearbejde") b = cp("b", "acme", "x", "DK", "Copenhagen, Denmark") - assert towns(a.place, "DK") == {"copenhagen"} + assert towns(a.place) == {"copenhagen"} assert same_posting(a, b) @@ -114,21 +115,35 @@ def test_country_rules(): ("Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", "DK", {"aalborg", "copenhagen", "kolding", "aarhus"}), ("Berlin und Munich", "DE", {"berlin", "munich"}), + ("Mountain View, California", "US", {"mountain view"}), + ("Ningbo, Zhejiang, China", "CN", {"ningbo"}), ("Germany", "DE", set()), ("Remote", "", set()), ("", "", set()), ]) def test_towns(place, country, expected): - assert towns(place, country) == expected + # A head `places` cannot parse, "remote berlin", may come along; it matches + # nothing, so only the towns that must be there and the empty case count. + found = towns(place) + assert found >= expected and (expected or not found) -def test_assign_is_transitive(): - # A matches B by Berlin, B matches C by Hamburg, A and C alone do not match. +def test_a_state_or_province_is_not_a_shared_town(): + # Live false merges before `places.towns`: Databricks and Bosch. + assert not same_posting(cp("a", country="US", place="Mountain View, California"), + cp("b", country="US", place="San Francisco, California")) + assert not same_posting(cp("a", country="CN", place="Ningbo, Zhejiang, China"), + cp("b", country="CN", place="Hangzhou, Zhejiang, China")) + + +def test_assign_does_not_chain(): + # A matches B by Berlin, B matches C by Hamburg, A and C do not match, so A + # cannot share C's group. a = cp("a", place="Berlin", posted="2026-09-03") b = cp("b", place="Berlin, Hamburg", posted="2026-09-02") c = cp("c", place="Hamburg", posted="2026-09-01") assert not same_posting(a, c) - assert assign([a, b, c]) == {"a": "c", "b": "c", "c": None} + assert assign([a, b, c]) == {"a": None, "b": "c", "c": None} def test_assign_returns_every_uid(): @@ -166,3 +181,16 @@ def test_a_row_without_a_town_joins_only_one_group(): assert assign(rows) == {"a": None, "b": "a", "c": "a"} rows = [cp("a", place=""), cp("b", place="Germany")] assert assign(rows) == {"a": None, "b": "a"} + + +def test_a_row_without_a_place_never_bridges_two_countries(): + # Live: LEGO "Sales Associate", three rows with no place, one Belgium, one + # Netherlands, `country` empty on all five until the place names one. + rows = [cp("a", country="", place=""), cp("b", country="", place=""), + cp("c", country="", place=""), cp("d", country="BE", place="Belgium"), + cp("e", country="NL", place="Netherlands")] + got = assign(rows) + assert got["d"] is None and got["e"] is None + assert got == {"a": None, "b": "a", "c": "a", "d": None, "e": None} + assert not same_posting(cp("a", country="EU", place="Belgium"), + cp("b", country="EU", place="Netherlands"))