Add the rule that decides which open roles are one posting
Pure module app/dupes.py: towns, same_posting, keep_order and assign, with tests on the live cross-host pairs, the SAP requisitions and the edge cases. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
de156ed90f
commit
d060bee5b7
2 files changed
+337
No files matched your search
+179
@@ -0,0 +1,179 @@
|
|||||||
|
"""Which open roles are one posting, and which copy is kept.
|
||||||
|
|
||||||
|
One job is often listed more than once: by a search provider and by the
|
||||||
|
company's own board, or by one board under several requisition ids (SAP lists
|
||||||
|
one job under 3 to 9 of them). The stored cluster key `(company_key,
|
||||||
|
title_key, place_key)` finds none of the cross-host pairs, because the places
|
||||||
|
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
|
||||||
|
`Lund, Skåne, Sweden`. So the rule here compares towns, not place strings.
|
||||||
|
|
||||||
|
Two rows are one posting when they have the same non-empty `company_key`, the
|
||||||
|
same `title_key`, the same `country` (or either has none), and the sets of
|
||||||
|
towns their places name share a member, where a row with no town matches
|
||||||
|
every town. Host does not matter. The same title in Berlin and in Munich
|
||||||
|
stays two roles, so a place filter still finds each. The join is transitive:
|
||||||
|
if A matches B and B matches C, all three are one group, although A and C
|
||||||
|
alone would not match (a row with no town bridges two towns for this reason).
|
||||||
|
|
||||||
|
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
|
||||||
|
it. Pure by design: no store, no SQL, no network, so the rule is tested on
|
||||||
|
the strings boards really write and the storage layer only feeds it rows.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from collections.abc import Iterable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from app.models import _COUNTRY_NAMES, _place_fold
|
||||||
|
|
||||||
|
#: Hosts that search across companies rather than run a company's own board.
|
||||||
|
#: A board closes a role reliably and carries the full text, so it is kept
|
||||||
|
#: over a copy found here.
|
||||||
|
SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"})
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Copy:
|
||||||
|
"""The columns of one open role the rule reads. `posted_at` is an ISO
|
||||||
|
string or None; `has_description` is whether `description` is non-empty."""
|
||||||
|
|
||||||
|
uid: str
|
||||||
|
company_key: str
|
||||||
|
title_key: str
|
||||||
|
country: str
|
||||||
|
place: str
|
||||||
|
host: str
|
||||||
|
has_description: bool
|
||||||
|
posted_at: str | None
|
||||||
|
|
||||||
|
|
||||||
|
#: Folded names of every country, in every language `_COUNTRY_NAMES` has.
|
||||||
|
_COUNTRIES: frozenset[str] = frozenset(
|
||||||
|
_place_fold(name)
|
||||||
|
for line in _COUNTRY_NAMES.split("\n") if line
|
||||||
|
for name in line.split(" ", 1)[1].split("|"))
|
||||||
|
|
||||||
|
#: Town names boards spell two ways, folded, to the one spelling used here.
|
||||||
|
_ALIASES: dict[str, str] = {
|
||||||
|
"kobenhavn": "copenhagen", "koebenhavn": "copenhagen", "munchen": "munich",
|
||||||
|
"koln": "cologne", "gothenburg": "goteborg", "wien": "vienna",
|
||||||
|
"praha": "prague", "warszawa": "warsaw",
|
||||||
|
}
|
||||||
|
|
||||||
|
#: Regions that follow a town and name no town of their own. Dropped only
|
||||||
|
#: when a town is left, so "Skåne" alone stays a place to compare.
|
||||||
|
_REGIONS: frozenset[str] = frozenset({
|
||||||
|
"skane", "scania", "sjaelland", "zealand", "hovedstaden", "capital region",
|
||||||
|
"midtjylland", "syddanmark", "nordjylland", "jylland", "stockholms lan",
|
||||||
|
"vastra gotaland", "uppsala lan", "oresund", "bavaria", "bayern",
|
||||||
|
"baden wurttemberg", "hessen", "nordrhein westfalen", "north rhine westphalia",
|
||||||
|
"england", "scotland", "wales", "emea", "europe", "eu", "dach", "nordics",
|
||||||
|
})
|
||||||
|
|
||||||
|
#: How the role is worked, not where. Removed from a part, so "Remote Berlin"
|
||||||
|
#: is Berlin, and a part that is only this is dropped.
|
||||||
|
_WORK = re.compile(
|
||||||
|
r"\b(?:hybrid|remote|position|mulighed for hjemmearbejde|hjemmearbejde|"
|
||||||
|
r"home ?office|work from home|on ?site|onsite|distributed|flexible)\b")
|
||||||
|
|
||||||
|
_SPLIT = re.compile(r"[,;|/()\n]")
|
||||||
|
_AND = re.compile(r" (?:og|och|und|and) ")
|
||||||
|
#: A postal district or code after a town: "kobenhavn k", "aarhus c", "2100".
|
||||||
|
_DISTRICT = re.compile(r"(?: (?:[a-z]|\d{2,5}))+$")
|
||||||
|
|
||||||
|
|
||||||
|
def towns(place: str, country: str) -> frozenset[str]:
|
||||||
|
"""The towns a place line names, folded: "Lund, Skåne, Sweden" is
|
||||||
|
{lund}, "København K og mulighed for hjemmearbejde" is {copenhagen}, and
|
||||||
|
"Houston, TX, United States" is {houston}. Empty when it names none.
|
||||||
|
|
||||||
|
The line splits at `,` `;` `|` `/` and parentheses, and each part at
|
||||||
|
` og `, ` och `, ` und ` and ` and `. Each is folded with
|
||||||
|
`models._place_fold`, then loses work-mode words, a trailing postal
|
||||||
|
district and its country's ISO code, and is dropped if it is a country
|
||||||
|
name, a region word or a two-letter US state code beside a town."""
|
||||||
|
parts: list[tuple[str, str]] = []
|
||||||
|
for raw in _SPLIT.split(place or ""):
|
||||||
|
raw = raw.strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
folded = _place_fold(raw)
|
||||||
|
pieces = [folded] if folded in _COUNTRIES else _AND.split(f" {folded} ")
|
||||||
|
parts += [(piece.strip(), raw) for piece in pieces]
|
||||||
|
found: set[str] = set()
|
||||||
|
regional: set[str] = set()
|
||||||
|
for piece, raw in parts:
|
||||||
|
if not piece or piece in _COUNTRIES or piece == _place_fold(country):
|
||||||
|
continue
|
||||||
|
piece = " ".join(_WORK.sub(" ", piece).split())
|
||||||
|
piece = _DISTRICT.sub("", piece) if " " in piece else piece
|
||||||
|
if not piece or piece in _COUNTRIES:
|
||||||
|
continue
|
||||||
|
if piece in _REGIONS or (len(piece) == 2 and raw.isupper()):
|
||||||
|
regional.add(piece)
|
||||||
|
continue
|
||||||
|
found.add(_ALIASES.get(piece, piece))
|
||||||
|
return frozenset(found or regional - {_place_fold(country)})
|
||||||
|
|
||||||
|
|
||||||
|
def same_posting(a: Copy, b: Copy) -> bool:
|
||||||
|
"""Whether two rows are one posting: a non-empty shared `company_key`,
|
||||||
|
the same `title_key`, compatible countries (equal, or either empty) and
|
||||||
|
towns that share a member or are empty on either side. Host is ignored."""
|
||||||
|
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
|
||||||
|
return False
|
||||||
|
if a.country and b.country and a.country != b.country:
|
||||||
|
return False
|
||||||
|
ta, tb = towns(a.place, a.country), towns(b.place, b.country)
|
||||||
|
return not ta or not tb or bool(ta & tb)
|
||||||
|
|
||||||
|
|
||||||
|
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
||||||
|
"""The sort key that picks the kept copy of a group, smallest first: a
|
||||||
|
board over a search provider, then one with a description, then the
|
||||||
|
earliest `posted_at` (none last), then the smallest `uid`, so the choice
|
||||||
|
is the same on every run."""
|
||||||
|
return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at,
|
||||||
|
c.posted_at or "", c.uid)
|
||||||
|
|
||||||
|
|
||||||
|
def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
|
||||||
|
"""`uid -> dup_of` for every row: the uid of its group's kept copy, None
|
||||||
|
for the kept copy itself and for a row with no copies.
|
||||||
|
|
||||||
|
Rows are partitioned by `(company_key, title_key)`, a row with no
|
||||||
|
`company_key` standing alone. Within a partition `same_posting` joins
|
||||||
|
rows transitively (union-find), and each component points every row but
|
||||||
|
its `keep_order` minimum at that minimum."""
|
||||||
|
result: dict[str, str | None] = {}
|
||||||
|
parts: dict[tuple[str, str], list[Copy]] = {}
|
||||||
|
for row in rows:
|
||||||
|
result[row.uid] = None
|
||||||
|
if row.company_key:
|
||||||
|
parts.setdefault((row.company_key, row.title_key), []).append(row)
|
||||||
|
for group in parts.values():
|
||||||
|
if len(group) < 2:
|
||||||
|
continue
|
||||||
|
parent = list(range(len(group)))
|
||||||
|
|
||||||
|
def find(i: int, parent: list[int] = parent) -> int:
|
||||||
|
while parent[i] != i:
|
||||||
|
parent[i] = parent[parent[i]]
|
||||||
|
i = parent[i]
|
||||||
|
return i
|
||||||
|
|
||||||
|
for i, a in enumerate(group):
|
||||||
|
for j in range(i + 1, len(group)):
|
||||||
|
if find(i) != find(j) and same_posting(a, group[j]):
|
||||||
|
parent[find(j)] = find(i)
|
||||||
|
comps: dict[int, list[Copy]] = {}
|
||||||
|
for i, row in enumerate(group):
|
||||||
|
comps.setdefault(find(i), []).append(row)
|
||||||
|
for comp in comps.values():
|
||||||
|
keeper = min(comp, key=keep_order)
|
||||||
|
for row in comp:
|
||||||
|
if row is not keeper:
|
||||||
|
result[row.uid] = keeper.uid
|
||||||
|
return result
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
"""`app.dupes` against postings as boards really wrote them. Pure: no store,
|
||||||
|
no network. The live cases are the 2026-09-29 database's."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
import pytest # noqa: E402
|
||||||
|
|
||||||
|
from app.dupes import Copy, assign, keep_order, same_posting, towns # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def cp(uid: str = "u", company: str = "acme", title: str = "engineer", country: str = "DE",
|
||||||
|
place: str = "", host: str = "greenhouse", desc: bool = True,
|
||||||
|
posted: str | None = None) -> Copy:
|
||||||
|
return Copy(uid, company, title, country, place, host, desc, posted)
|
||||||
|
|
||||||
|
|
||||||
|
#: `(company_key, title_key, country, host_a, place_a, host_b, place_b, kept host)`.
|
||||||
|
LIVE = [
|
||||||
|
("ericsson", "entry level 6g developer", "SE",
|
||||||
|
"platsbanken", "Lund", "eightfold", "Lund, Skåne, Sweden", "eightfold"),
|
||||||
|
("gn-group", "machine learning engineer", "DK",
|
||||||
|
"jobindex", "Ballerup", "workday", "Ballerup, Denmark", "workday"),
|
||||||
|
("kmd", "student worker finance", "DK",
|
||||||
|
"jobindex", "Ballerup, Ballerup, Aalborg, Aalborg", "successfactors", "Ballerup, Denmark",
|
||||||
|
"successfactors"),
|
||||||
|
("trackunit", "machine learning engineer", "DK",
|
||||||
|
"jobindex", "Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position",
|
||||||
|
"teamtailor", "København, Denmark; Aarhus, Denmark; Kolding, Denmark; Aalborg, Denmark",
|
||||||
|
"teamtailor"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("company,title,country,ha,pa,hb,pb,kept", LIVE)
|
||||||
|
def test_live_cross_host_pairs(company, title, country, ha, pa, hb, pb, kept):
|
||||||
|
a = cp("a", company, title, country, pa, ha)
|
||||||
|
b = cp("b", company, title, country, pb, hb)
|
||||||
|
assert same_posting(a, b) and same_posting(b, a)
|
||||||
|
result = assign([a, b])
|
||||||
|
assert result == ({"a": "b", "b": None} if kept == hb else {"a": None, "b": "a"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_sap_requisitions():
|
||||||
|
place = "Garching bei München (Munich), Germany"
|
||||||
|
title = "forward deployed chief ai engineering expert"
|
||||||
|
rows = [
|
||||||
|
cp("g3", "sap", title, "DE", place, "successfactors", False, "2026-09-02"),
|
||||||
|
cp("g1", "sap", title, "DE", place, "successfactors", False, "2026-09-01"),
|
||||||
|
cp("g2", "sap", title, "DE", place, "successfactors", False, "2026-09-01"),
|
||||||
|
cp("ny", "sap", title, "US", "New York, United States", "successfactors", True,
|
||||||
|
"2026-08-01"),
|
||||||
|
]
|
||||||
|
# The earliest posted_at, then the smallest uid, is kept; New York is alone
|
||||||
|
# although it is the only row with a description.
|
||||||
|
assert assign(rows) == {"g1": None, "g2": "g1", "g3": "g1", "ny": None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_houston_state_code():
|
||||||
|
a = cp("a", "sap", "x", "US", "Houston, TX, United States")
|
||||||
|
b = cp("b", "sap", "x", "US", "Houston, United States")
|
||||||
|
assert same_posting(a, b)
|
||||||
|
|
||||||
|
|
||||||
|
def test_copenhagen_district_and_remote_phrase():
|
||||||
|
a = cp("a", "acme", "x", "DK", "København K og mulighed for hjemmearbejde")
|
||||||
|
b = cp("b", "acme", "x", "DK", "Copenhagen, Denmark")
|
||||||
|
assert towns(a.place, "DK") == {"copenhagen"}
|
||||||
|
assert same_posting(a, b)
|
||||||
|
|
||||||
|
|
||||||
|
def test_different_towns_are_not_the_same():
|
||||||
|
assert not same_posting(cp("a", place="Berlin, Germany"), cp("b", place="Munich, Germany"))
|
||||||
|
assert assign([cp("a", place="Berlin"), cp("b", place="Munich")]) == {"a": None, "b": None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_place_matches():
|
||||||
|
assert same_posting(cp("a", place=""), cp("b", place="Berlin"))
|
||||||
|
assert same_posting(cp("a", place=""), cp("b", place=""))
|
||||||
|
assert same_posting(cp("a", place="Germany"), cp("b", place="Berlin, Germany"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_company_never_grouped():
|
||||||
|
a, b = cp("a", company=""), cp("b", company="")
|
||||||
|
assert not same_posting(a, b)
|
||||||
|
assert assign([a, b]) == {"a": None, "b": None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_other_company_or_title_is_not_the_same():
|
||||||
|
assert not same_posting(cp("a"), cp("b", company="other"))
|
||||||
|
assert not same_posting(cp("a"), cp("b", title="designer"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_country_rules():
|
||||||
|
assert not same_posting(cp("a", country="DE", place="Berlin"),
|
||||||
|
cp("b", country="US", place="Berlin"))
|
||||||
|
assert same_posting(cp("a", country="", place="Berlin"),
|
||||||
|
cp("b", country="DE", place="Berlin, Germany"))
|
||||||
|
assert not same_posting(cp("a", country="", place="Berlin"),
|
||||||
|
cp("b", country="DE", place="Munich"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("place,country,expected", [
|
||||||
|
("Lund, Skåne, Sweden", "SE", {"lund"}),
|
||||||
|
("Houston, TX, United States", "US", {"houston"}),
|
||||||
|
("New York, United States", "US", {"new york"}),
|
||||||
|
("Aarhus C", "DK", {"aarhus"}),
|
||||||
|
("Remote Berlin", "DE", {"berlin"}),
|
||||||
|
("Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", "DK",
|
||||||
|
{"aalborg", "copenhagen", "kolding", "aarhus"}),
|
||||||
|
("Berlin und Munich", "DE", {"berlin", "munich"}),
|
||||||
|
("Germany", "DE", set()),
|
||||||
|
("Remote", "", set()),
|
||||||
|
("", "", set()),
|
||||||
|
])
|
||||||
|
def test_towns(place, country, expected):
|
||||||
|
assert towns(place, country) == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_assign_is_transitive():
|
||||||
|
# A matches B by Berlin, B matches C by Hamburg, A and C alone do not match.
|
||||||
|
a = cp("a", place="Berlin", posted="2026-09-03")
|
||||||
|
b = cp("b", place="Berlin, Hamburg", posted="2026-09-02")
|
||||||
|
c = cp("c", place="Hamburg", posted="2026-09-01")
|
||||||
|
assert not same_posting(a, c)
|
||||||
|
assert assign([a, b, c]) == {"a": "c", "b": "c", "c": None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_assign_returns_every_uid():
|
||||||
|
rows = [cp("a"), cp("b", title="other"), cp("c", company="")]
|
||||||
|
assert assign(rows) == {"a": None, "b": None, "c": None}
|
||||||
|
assert assign([]) == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_assign_separate_groups_in_one_partition():
|
||||||
|
rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="Munich"),
|
||||||
|
cp("d", place="Munich")]
|
||||||
|
assert assign(rows) == {"a": None, "b": "a", "c": None, "d": "c"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_keep_order_ranks():
|
||||||
|
board = cp("z", host="workday", desc=False, posted="2026-09-09")
|
||||||
|
search = cp("a", host="jobindex", desc=True, posted="2026-01-01")
|
||||||
|
assert min([search, board], key=keep_order) is board
|
||||||
|
with_desc = cp("z", desc=True, posted="2026-09-09")
|
||||||
|
without = cp("a", desc=False, posted="2026-01-01")
|
||||||
|
assert min([without, with_desc], key=keep_order) is with_desc
|
||||||
|
early = cp("z", posted="2026-01-01")
|
||||||
|
late = cp("a", posted="2026-02-01")
|
||||||
|
undated = cp("0", posted=None)
|
||||||
|
assert sorted([undated, late, early], key=keep_order) == [early, late, undated]
|
||||||
|
assert min([cp("b", posted="2026-01-01"), cp("a", posted="2026-01-01")],
|
||||||
|
key=keep_order).uid == "a"
|
||||||
Reference in new issue
Block a user