Add the rule that decides which open roles are one posting

Pure module app/dupes.py: towns, same_posting, keep_order and assign, with
tests on the live cross-host pairs, the SAP requisitions and the edge cases.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-09-29 22:06:08 +02:00
1 parent de156ed90f
commit d060bee5b7
2 files changed
+337

No files matched your search

+179
View File
@@ -0,0 +1,179 @@
"""Which open roles are one posting, and which copy is kept.
One job is often listed more than once: by a search provider and by the
company's own board, or by one board under several requisition ids (SAP lists
one job under 3 to 9 of them). The stored cluster key `(company_key,
title_key, place_key)` finds none of the cross-host pairs, because the places
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
`Lund, Skåne, Sweden`. So the rule here compares towns, not place strings.
Two rows are one posting when they have the same non-empty `company_key`, the
same `title_key`, the same `country` (or either has none), and the sets of
towns their places name share a member, where a row with no town matches
every town. Host does not matter. The same title in Berlin and in Munich
stays two roles, so a place filter still finds each. The join is transitive:
if A matches B and B matches C, all three are one group, although A and C
alone would not match (a row with no town bridges two towns for this reason).
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
it. Pure by design: no store, no SQL, no network, so the rule is tested on
the strings boards really write and the storage layer only feeds it rows.
"""
from __future__ import annotations
import re
from collections.abc import Iterable
from dataclasses import dataclass
from app.models import _COUNTRY_NAMES, _place_fold
#: Hosts that search across companies rather than run a company's own board.
#: A board closes a role reliably and carries the full text, so it is kept
#: over a copy found here.
SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"})
@dataclass(frozen=True)
class Copy:
"""The columns of one open role the rule reads. `posted_at` is an ISO
string or None; `has_description` is whether `description` is non-empty."""
uid: str
company_key: str
title_key: str
country: str
place: str
host: str
has_description: bool
posted_at: str | None
#: Folded names of every country, in every language `_COUNTRY_NAMES` has.
_COUNTRIES: frozenset[str] = frozenset(
_place_fold(name)
for line in _COUNTRY_NAMES.split("\n") if line
for name in line.split(" ", 1)[1].split("|"))
#: Town names boards spell two ways, folded, to the one spelling used here.
_ALIASES: dict[str, str] = {
"kobenhavn": "copenhagen", "koebenhavn": "copenhagen", "munchen": "munich",
"koln": "cologne", "gothenburg": "goteborg", "wien": "vienna",
"praha": "prague", "warszawa": "warsaw",
}
#: Regions that follow a town and name no town of their own. Dropped only
#: when a town is left, so "Skåne" alone stays a place to compare.
_REGIONS: frozenset[str] = frozenset({
"skane", "scania", "sjaelland", "zealand", "hovedstaden", "capital region",
"midtjylland", "syddanmark", "nordjylland", "jylland", "stockholms lan",
"vastra gotaland", "uppsala lan", "oresund", "bavaria", "bayern",
"baden wurttemberg", "hessen", "nordrhein westfalen", "north rhine westphalia",
"england", "scotland", "wales", "emea", "europe", "eu", "dach", "nordics",
})
#: How the role is worked, not where. Removed from a part, so "Remote Berlin"
#: is Berlin, and a part that is only this is dropped.
_WORK = re.compile(
r"\b(?:hybrid|remote|position|mulighed for hjemmearbejde|hjemmearbejde|"
r"home ?office|work from home|on ?site|onsite|distributed|flexible)\b")
_SPLIT = re.compile(r"[,;|/()\n]")
_AND = re.compile(r" (?:og|och|und|and) ")
#: A postal district or code after a town: "kobenhavn k", "aarhus c", "2100".
_DISTRICT = re.compile(r"(?: (?:[a-z]|\d{2,5}))+$")
def towns(place: str, country: str) -> frozenset[str]:
"""The towns a place line names, folded: "Lund, Skåne, Sweden" is
{lund}, "København K og mulighed for hjemmearbejde" is {copenhagen}, and
"Houston, TX, United States" is {houston}. Empty when it names none.
The line splits at `,` `;` `|` `/` and parentheses, and each part at
` og `, ` och `, ` und ` and ` and `. Each is folded with
`models._place_fold`, then loses work-mode words, a trailing postal
district and its country's ISO code, and is dropped if it is a country
name, a region word or a two-letter US state code beside a town."""
parts: list[tuple[str, str]] = []
for raw in _SPLIT.split(place or ""):
raw = raw.strip()
if not raw:
continue
folded = _place_fold(raw)
pieces = [folded] if folded in _COUNTRIES else _AND.split(f" {folded} ")
parts += [(piece.strip(), raw) for piece in pieces]
found: set[str] = set()
regional: set[str] = set()
for piece, raw in parts:
if not piece or piece in _COUNTRIES or piece == _place_fold(country):
continue
piece = " ".join(_WORK.sub(" ", piece).split())
piece = _DISTRICT.sub("", piece) if " " in piece else piece
if not piece or piece in _COUNTRIES:
continue
if piece in _REGIONS or (len(piece) == 2 and raw.isupper()):
regional.add(piece)
continue
found.add(_ALIASES.get(piece, piece))
return frozenset(found or regional - {_place_fold(country)})
def same_posting(a: Copy, b: Copy) -> bool:
"""Whether two rows are one posting: a non-empty shared `company_key`,
the same `title_key`, compatible countries (equal, or either empty) and
towns that share a member or are empty on either side. Host is ignored."""
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
return False
if a.country and b.country and a.country != b.country:
return False
ta, tb = towns(a.place, a.country), towns(b.place, b.country)
return not ta or not tb or bool(ta & tb)
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
"""The sort key that picks the kept copy of a group, smallest first: a
board over a search provider, then one with a description, then the
earliest `posted_at` (none last), then the smallest `uid`, so the choice
is the same on every run."""
return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at,
c.posted_at or "", c.uid)
def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
"""`uid -> dup_of` for every row: the uid of its group's kept copy, None
for the kept copy itself and for a row with no copies.
Rows are partitioned by `(company_key, title_key)`, a row with no
`company_key` standing alone. Within a partition `same_posting` joins
rows transitively (union-find), and each component points every row but
its `keep_order` minimum at that minimum."""
result: dict[str, str | None] = {}
parts: dict[tuple[str, str], list[Copy]] = {}
for row in rows:
result[row.uid] = None
if row.company_key:
parts.setdefault((row.company_key, row.title_key), []).append(row)
for group in parts.values():
if len(group) < 2:
continue
parent = list(range(len(group)))
def find(i: int, parent: list[int] = parent) -> int:
while parent[i] != i:
parent[i] = parent[parent[i]]
i = parent[i]
return i
for i, a in enumerate(group):
for j in range(i + 1, len(group)):
if find(i) != find(j) and same_posting(a, group[j]):
parent[find(j)] = find(i)
comps: dict[int, list[Copy]] = {}
for i, row in enumerate(group):
comps.setdefault(find(i), []).append(row)
for comp in comps.values():
keeper = min(comp, key=keep_order)
for row in comp:
if row is not keeper:
result[row.uid] = keeper.uid
return result
+158
View File
@@ -0,0 +1,158 @@
"""`app.dupes` against postings as boards really wrote them. Pure: no store,
no network. The live cases are the 2026-09-29 database's."""
from __future__ import annotations
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
import pytest # noqa: E402
from app.dupes import Copy, assign, keep_order, same_posting, towns # noqa: E402
def cp(uid: str = "u", company: str = "acme", title: str = "engineer", country: str = "DE",
place: str = "", host: str = "greenhouse", desc: bool = True,
posted: str | None = None) -> Copy:
return Copy(uid, company, title, country, place, host, desc, posted)
#: `(company_key, title_key, country, host_a, place_a, host_b, place_b, kept host)`.
LIVE = [
("ericsson", "entry level 6g developer", "SE",
"platsbanken", "Lund", "eightfold", "Lund, Skåne, Sweden", "eightfold"),
("gn-group", "machine learning engineer", "DK",
"jobindex", "Ballerup", "workday", "Ballerup, Denmark", "workday"),
("kmd", "student worker finance", "DK",
"jobindex", "Ballerup, Ballerup, Aalborg, Aalborg", "successfactors", "Ballerup, Denmark",
"successfactors"),
("trackunit", "machine learning engineer", "DK",
"jobindex", "Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position",
"teamtailor", "København, Denmark; Aarhus, Denmark; Kolding, Denmark; Aalborg, Denmark",
"teamtailor"),
]
@pytest.mark.parametrize("company,title,country,ha,pa,hb,pb,kept", LIVE)
def test_live_cross_host_pairs(company, title, country, ha, pa, hb, pb, kept):
a = cp("a", company, title, country, pa, ha)
b = cp("b", company, title, country, pb, hb)
assert same_posting(a, b) and same_posting(b, a)
result = assign([a, b])
assert result == ({"a": "b", "b": None} if kept == hb else {"a": None, "b": "a"})
def test_sap_requisitions():
place = "Garching bei München (Munich), Germany"
title = "forward deployed chief ai engineering expert"
rows = [
cp("g3", "sap", title, "DE", place, "successfactors", False, "2026-09-02"),
cp("g1", "sap", title, "DE", place, "successfactors", False, "2026-09-01"),
cp("g2", "sap", title, "DE", place, "successfactors", False, "2026-09-01"),
cp("ny", "sap", title, "US", "New York, United States", "successfactors", True,
"2026-08-01"),
]
# The earliest posted_at, then the smallest uid, is kept; New York is alone
# although it is the only row with a description.
assert assign(rows) == {"g1": None, "g2": "g1", "g3": "g1", "ny": None}
def test_houston_state_code():
a = cp("a", "sap", "x", "US", "Houston, TX, United States")
b = cp("b", "sap", "x", "US", "Houston, United States")
assert same_posting(a, b)
def test_copenhagen_district_and_remote_phrase():
a = cp("a", "acme", "x", "DK", "København K og mulighed for hjemmearbejde")
b = cp("b", "acme", "x", "DK", "Copenhagen, Denmark")
assert towns(a.place, "DK") == {"copenhagen"}
assert same_posting(a, b)
def test_different_towns_are_not_the_same():
assert not same_posting(cp("a", place="Berlin, Germany"), cp("b", place="Munich, Germany"))
assert assign([cp("a", place="Berlin"), cp("b", place="Munich")]) == {"a": None, "b": None}
def test_empty_place_matches():
assert same_posting(cp("a", place=""), cp("b", place="Berlin"))
assert same_posting(cp("a", place=""), cp("b", place=""))
assert same_posting(cp("a", place="Germany"), cp("b", place="Berlin, Germany"))
def test_empty_company_never_grouped():
a, b = cp("a", company=""), cp("b", company="")
assert not same_posting(a, b)
assert assign([a, b]) == {"a": None, "b": None}
def test_other_company_or_title_is_not_the_same():
assert not same_posting(cp("a"), cp("b", company="other"))
assert not same_posting(cp("a"), cp("b", title="designer"))
def test_country_rules():
assert not same_posting(cp("a", country="DE", place="Berlin"),
cp("b", country="US", place="Berlin"))
assert same_posting(cp("a", country="", place="Berlin"),
cp("b", country="DE", place="Berlin, Germany"))
assert not same_posting(cp("a", country="", place="Berlin"),
cp("b", country="DE", place="Munich"))
@pytest.mark.parametrize("place,country,expected", [
("Lund, Skåne, Sweden", "SE", {"lund"}),
("Houston, TX, United States", "US", {"houston"}),
("New York, United States", "US", {"new york"}),
("Aarhus C", "DK", {"aarhus"}),
("Remote Berlin", "DE", {"berlin"}),
("Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", "DK",
{"aalborg", "copenhagen", "kolding", "aarhus"}),
("Berlin und Munich", "DE", {"berlin", "munich"}),
("Germany", "DE", set()),
("Remote", "", set()),
("", "", set()),
])
def test_towns(place, country, expected):
assert towns(place, country) == expected
def test_assign_is_transitive():
# A matches B by Berlin, B matches C by Hamburg, A and C alone do not match.
a = cp("a", place="Berlin", posted="2026-09-03")
b = cp("b", place="Berlin, Hamburg", posted="2026-09-02")
c = cp("c", place="Hamburg", posted="2026-09-01")
assert not same_posting(a, c)
assert assign([a, b, c]) == {"a": "c", "b": "c", "c": None}
def test_assign_returns_every_uid():
rows = [cp("a"), cp("b", title="other"), cp("c", company="")]
assert assign(rows) == {"a": None, "b": None, "c": None}
assert assign([]) == {}
def test_assign_separate_groups_in_one_partition():
rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="Munich"),
cp("d", place="Munich")]
assert assign(rows) == {"a": None, "b": "a", "c": None, "d": "c"}
def test_keep_order_ranks():
board = cp("z", host="workday", desc=False, posted="2026-09-09")
search = cp("a", host="jobindex", desc=True, posted="2026-01-01")
assert min([search, board], key=keep_order) is board
with_desc = cp("z", desc=True, posted="2026-09-09")
without = cp("a", desc=False, posted="2026-01-01")
assert min([without, with_desc], key=keep_order) is with_desc
early = cp("z", posted="2026-01-01")
late = cp("a", posted="2026-02-01")
undated = cp("0", posted=None)
assert sorted([undated, late, early], key=keep_order) == [early, late, undated]
assert min([cp("b", posted="2026-01-01"), cp("a", posted="2026-01-01")],
key=keep_order).uid == "a"