From 1e22537ae500132ddf73022786aba082e4adbd9c Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 22:07:05 +0200 Subject: [PATCH] Let a role that names no town join one group, never bridge two, and place dupes above places Co-Authored-By: Claude Opus 5.5 --- app/dupes.py | 32 +++++++++++++++++++++++++------- tests/test_architecture.py | 2 +- tests/test_dupes.py | 10 ++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/app/dupes.py b/app/dupes.py index f08d10a..9fdeb9b 100644 --- a/app/dupes.py +++ b/app/dupes.py @@ -11,9 +11,10 @@ Two rows are one posting when they have the same non-empty `company_key`, the same `title_key`, the same `country` (or either has none), and the sets of towns their places name share a member, where a row with no town matches every town. Host does not matter. The same title in Berlin and in Munich -stays two roles, so a place filter still finds each. The join is transitive: -if A matches B and B matches C, all three are one group, although A and C -alone would not match (a row with no town bridges two towns for this reason). +stays two roles, so a place filter still finds each. The join is transitive +among rows that name a town. A row that names none joins a group only when +exactly one group of its company and title would take it, so a bare +"Germany" never merges Berlin and Munich into one role. Each group keeps one copy (`keep_order`, smallest wins) and the rest point at it. Pure by design: no store, no SQL, no network, so the rule is tested on @@ -144,9 +145,10 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]: for the kept copy itself and for a row with no copies. Rows are partitioned by `(company_key, title_key)`, a row with no - `company_key` standing alone. Within a partition `same_posting` joins - rows transitively (union-find), and each component points every row but - its `keep_order` minimum at that minimum.""" + `company_key` standing alone. Within a partition `same_posting` joins the + rows that name a town transitively (union-find); a row naming none joins + the one group it matches, or stays out when it matches two. Each + component points every row but its `keep_order` minimum at that minimum.""" result: dict[str, str | None] = {} parts: dict[tuple[str, str], list[Copy]] = {} for row in rows: @@ -164,10 +166,26 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]: i = parent[i] return i + placed = [bool(towns(row.place, row.country)) for row in group] for i, a in enumerate(group): for j in range(i + 1, len(group)): - if find(i) != find(j) and same_posting(a, group[j]): + if (placed[i] and placed[j] and find(i) != find(j) + and same_posting(a, group[j])): parent[find(j)] = find(i) + # A row without a town matches every town, so it would bridge two + # groups it matches; it joins only when there is one to join. Rows + # without a town that fit no placed group group among themselves. + loose = [i for i in range(len(group)) if not placed[i]] + for i in loose: + roots = {find(j) for j in range(len(group)) + if placed[j] and same_posting(group[i], group[j])} + if len(roots) == 1: + parent[i] = roots.pop() + for n, i in enumerate(loose): + for j in loose[n + 1:]: + if (parent[i] == i and parent[j] == j + and same_posting(group[i], group[j])): + parent[j] = i comps: dict[int, list[Copy]] = {} for i, row in enumerate(group): comps.setdefault(find(i), []).append(row) diff --git a/tests/test_architecture.py b/tests/test_architecture.py index 78f0b37..c3dc326 100644 --- a/tests/test_architecture.py +++ b/tests/test_architecture.py @@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1] #: no database, so storage may use them. `main` assembles the app from #: everything below it; `cli` and `__main__` are the entry points and may #: reach anything. -LAYERS = ["models", "places", "seniority", "config", "storage", "jobs", "web", "main", "cli", +LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli", "__main__"] APP = "app" EXTERNAL = {"desk", "pipeline", "auth"} diff --git a/tests/test_dupes.py b/tests/test_dupes.py index 4073baf..82f379c 100644 --- a/tests/test_dupes.py +++ b/tests/test_dupes.py @@ -156,3 +156,13 @@ def test_keep_order_ranks(): assert sorted([undated, late, early], key=keep_order) == [early, late, undated] assert min([cp("b", posted="2026-01-01"), cp("a", posted="2026-01-01")], key=keep_order).uid == "a" + + +def test_a_row_without_a_town_joins_only_one_group(): + # "Germany" matches Berlin and Munich; joining both would make them one role. + rows = [cp("a", place="Berlin"), cp("b", place="Munich"), cp("c", place="Germany")] + assert assign(rows) == {"a": None, "b": None, "c": None} + rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="")] + assert assign(rows) == {"a": None, "b": "a", "c": "a"} + rows = [cp("a", place=""), cp("b", place="Germany")] + assert assign(rows) == {"a": None, "b": "a"}