Group copies only when they all match, compare towns through places.towns, and record the rule in the plan

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 22:12:39 +02:00
1 parent fdb6264c98
commit b09a2f6b33
4 files changed
+90 -134

No files matched your search

+38 -115
View File
@@ -5,16 +5,16 @@ company's own board, or by one board under several requisition ids (SAP lists
one job under 3 to 9 of them). The stored cluster key `(company_key,
title_key, place_key)` finds none of the cross-host pairs, because the places
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
`Lund, Skåne, Sweden`. So the rule here compares towns, not place strings.
`Lund, Skåne, Sweden`. So the rule here compares towns as `places.towns` reads
them, not place strings.
Two rows are one posting when they have the same non-empty `company_key`, the
same `title_key`, the same `country` (or either has none), and the sets of
towns their places name share a member, where a row with no town matches
every town. Host does not matter. The same title in Berlin and in Munich
stays two roles, so a place filter still finds each. The join is transitive
among rows that name a town. A row that names none joins a group only when
exactly one group of its company and title would take it, so a bare
"Germany" never merges Berlin and Munich into one role.
stays two roles, so a place filter still finds each. A group is a set of
rows that each match all the others, never a chain: a row with no place
matches Belgium and the Netherlands, and must not make them one role.
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
it. Pure by design: no store, no SQL, no network, so the rule is tested on
@@ -23,11 +23,11 @@ the strings boards really write and the storage layer only feeds it rows.
from __future__ import annotations
import re
from collections.abc import Iterable
from dataclasses import dataclass
from app.models import _COUNTRY_NAMES, _place_fold
from app import places
from app.models import country_of
#: Hosts that search across companies rather than run a company's own board.
#: A board closes a role reliably and carries the full text, so it is kept
@@ -50,73 +50,16 @@ class Copy:
posted_at: str | None
#: Folded names of every country, in every language `_COUNTRY_NAMES` has.
_COUNTRIES: frozenset[str] = frozenset(
_place_fold(name)
for line in _COUNTRY_NAMES.split("\n") if line
for name in line.split(" ", 1)[1].split("|"))
#: Town names boards spell two ways, folded, to the one spelling used here.
_ALIASES: dict[str, str] = {
"kobenhavn": "copenhagen", "koebenhavn": "copenhagen", "munchen": "munich",
"koln": "cologne", "gothenburg": "goteborg", "wien": "vienna",
"praha": "prague", "warszawa": "warsaw",
}
#: Regions that follow a town and name no town of their own. Dropped only
#: when a town is left, so "Skåne" alone stays a place to compare.
_REGIONS: frozenset[str] = frozenset({
"skane", "scania", "sjaelland", "zealand", "hovedstaden", "capital region",
"midtjylland", "syddanmark", "nordjylland", "jylland", "stockholms lan",
"vastra gotaland", "uppsala lan", "oresund", "bavaria", "bayern",
"baden wurttemberg", "hessen", "nordrhein westfalen", "north rhine westphalia",
"england", "scotland", "wales", "emea", "europe", "eu", "dach", "nordics",
})
#: How the role is worked, not where. Removed from a part, so "Remote Berlin"
#: is Berlin, and a part that is only this is dropped.
_WORK = re.compile(
r"\b(?:hybrid|remote|position|mulighed for hjemmearbejde|hjemmearbejde|"
r"home ?office|work from home|on ?site|onsite|distributed|flexible)\b")
_SPLIT = re.compile(r"[,;|/()\n]")
_AND = re.compile(r" (?:og|och|und|and) ")
#: A postal district or code after a town: "kobenhavn k", "aarhus c", "2100".
_DISTRICT = re.compile(r"(?: (?:[a-z]|\d{2,5}))+$")
def _country(c: Copy) -> str:
"""The row's country, or the one its place names when the column is empty
or `EU`, a board's word for several offices: "Netherlands" is NL."""
return c.country if c.country not in ("", "EU") else country_of(c.place)
def towns(place: str, country: str) -> frozenset[str]:
"""The towns a place line names, folded: "Lund, Skåne, Sweden" is
{lund}, "København K og mulighed for hjemmearbejde" is {copenhagen}, and
"Houston, TX, United States" is {houston}. Empty when it names none.
The line splits at `,` `;` `|` `/` and parentheses, and each part at
` og `, ` och `, ` und ` and ` and `. Each is folded with
`models._place_fold`, then loses work-mode words, a trailing postal
district and its country's ISO code, and is dropped if it is a country
name, a region word or a two-letter US state code beside a town."""
parts: list[tuple[str, str]] = []
for raw in _SPLIT.split(place or ""):
raw = raw.strip()
if not raw:
continue
folded = _place_fold(raw)
pieces = [folded] if folded in _COUNTRIES else _AND.split(f" {folded} ")
parts += [(piece.strip(), raw) for piece in pieces]
found: set[str] = set()
regional: set[str] = set()
for piece, raw in parts:
if not piece or piece in _COUNTRIES or piece == _place_fold(country):
continue
piece = " ".join(_WORK.sub(" ", piece).split())
piece = _DISTRICT.sub("", piece) if " " in piece else piece
if not piece or piece in _COUNTRIES:
continue
if piece in _REGIONS or (len(piece) == 2 and raw.isupper()):
regional.add(piece)
continue
found.add(_ALIASES.get(piece, piece))
return frozenset(found or regional - {_place_fold(country)})
def _tier(c: Copy) -> int:
"""How precisely a row says where: 0 names a town, 1 only a country, 2
neither."""
return 0 if places.towns(c.place) else 1 if _country(c) else 2
def same_posting(a: Copy, b: Copy) -> bool:
@@ -125,9 +68,10 @@ def same_posting(a: Copy, b: Copy) -> bool:
towns that share a member or are empty on either side. Host is ignored."""
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
return False
if a.country and b.country and a.country != b.country:
ca, cb = _country(a), _country(b)
if ca and cb and ca != cb:
return False
ta, tb = towns(a.place, a.country), towns(b.place, b.country)
ta, tb = places.towns(a.place), places.towns(b.place)
return not ta or not tb or bool(ta & tb)
@@ -145,53 +89,32 @@ def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
for the kept copy itself and for a row with no copies.
Rows are partitioned by `(company_key, title_key)`, a row with no
`company_key` standing alone. Within a partition `same_posting` joins the
rows that name a town transitively (union-find); a row naming none joins
the one group it matches, or stays out when it matches two. Each
component points every row but its `keep_order` minimum at that minimum."""
`company_key` standing alone. Within a partition rows are taken sharpest
tier first and in `keep_order`, and a row joins a group only when it is `same_posting` with every member.
Joining on one match would chain: a row with no country matches Japan and
Malaysia both and would make them one. A row naming a town that fits two
groups joins the first. A vaguer row joins the first group of its own tier
it fits, else the one sharper group it fits, else starts its own: with no
place it fits Belgium and the Netherlands, and joins neither. Each group keeps its `keep_order`
minimum, so a board with no place still wins over a search with a town."""
result: dict[str, str | None] = {}
parts: dict[tuple[str, str], list[Copy]] = {}
for row in rows:
result[row.uid] = None
if row.company_key:
parts.setdefault((row.company_key, row.title_key), []).append(row)
for group in parts.values():
if len(group) < 2:
continue
parent = list(range(len(group)))
def find(i: int, parent: list[int] = parent) -> int:
while parent[i] != i:
parent[i] = parent[parent[i]]
i = parent[i]
return i
placed = [bool(towns(row.place, row.country)) for row in group]
for i, a in enumerate(group):
for j in range(i + 1, len(group)):
if (placed[i] and placed[j] and find(i) != find(j)
and same_posting(a, group[j])):
parent[find(j)] = find(i)
# A row without a town matches every town, so it would bridge two
# groups it matches; it joins only when there is one to join. Rows
# without a town that fit no placed group group among themselves.
loose = [i for i in range(len(group)) if not placed[i]]
for i in loose:
roots = {find(j) for j in range(len(group))
if placed[j] and same_posting(group[i], group[j])}
if len(roots) == 1:
parent[i] = roots.pop()
for n, i in enumerate(loose):
for j in loose[n + 1:]:
if (parent[i] == i and parent[j] == j
and same_posting(group[i], group[j])):
parent[j] = i
comps: dict[int, list[Copy]] = {}
for i, row in enumerate(group):
comps.setdefault(find(i), []).append(row)
for comp in comps.values():
keeper = min(comp, key=keep_order)
for row in comp:
for part in parts.values():
groups: list[list[Copy]] = []
for row in sorted(part, key=lambda r: (_tier(r), keep_order(r))):
fits = [g for g in groups if all(same_posting(row, m) for m in g)]
level = [g for g in fits if _tier(g[0]) == _tier(row)]
if level or len(fits) == 1:
(level or fits)[0].append(row)
else:
groups.append([row])
for group in groups:
keeper = min(group, key=keep_order)
for row in group:
if row is not keeper:
result[row.uid] = keeper.uid
return result
+14 -1
View File
@@ -35,7 +35,7 @@ from functools import lru_cache
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
"label", "match_sql", "matches", "parse", "register", "suggest"]
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
def fold(text: str) -> str:
@@ -578,6 +578,19 @@ def _towns_in(folded: str) -> list[str]:
return list(dict.fromkeys(found))
def towns(place: str) -> frozenset[str]:
"""The towns a line names, each as its group's key, so `København K` and
`Copenhagen, Denmark` are both `{"copenhagen"}`: the first comma part of each
place, and every known town the line holds as whole words, so a list such as
`Aalborg, Copenhagen, Kolding` gives all three. A state or province after
the town is not a town, so `Mountain View, California` and `San Francisco,
California` share none. Empty for a line that names only a country, a region
or how the role is worked."""
found = {_group(fold(town)) for town in _towns(place)}
found.update(_towns_in(fold(place or "")))
return frozenset(found)
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
"""Open roles per region, over folded lines. A line is cut into every run of
words once, and each region asks whether it holds one of its phrases."""