Merge branch 'wp-h-dupes'

This commit is contained in:
Wolf committed 2026-09-29 22:12:39 +02:00
commit 93bdad72a2
5 files changed
+335 -14

No files matched your search

+120
View File
@@ -0,0 +1,120 @@
"""Which open roles are one posting, and which copy is kept.
One job is often listed more than once: by a search provider and by the
company's own board, or by one board under several requisition ids (SAP lists
one job under 3 to 9 of them). The stored cluster key `(company_key,
title_key, place_key)` finds none of the cross-host pairs, because the places
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
`Lund, Skåne, Sweden`. So the rule here compares towns as `places.towns` reads
them, not place strings.
Two rows are one posting when they have the same non-empty `company_key`, the
same `title_key`, the same `country` (or either has none), and the sets of
towns their places name share a member, where a row with no town matches
every town. Host does not matter. The same title in Berlin and in Munich
stays two roles, so a place filter still finds each. A group is a set of
rows that each match all the others, never a chain: a row with no place
matches Belgium and the Netherlands, and must not make them one role.
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
it. Pure by design: no store, no SQL, no network, so the rule is tested on
the strings boards really write and the storage layer only feeds it rows.
"""
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
from app import places
from app.models import country_of
#: Hosts that search across companies rather than run a company's own board.
#: A board closes a role reliably and carries the full text, so it is kept
#: over a copy found here.
SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"})
@dataclass(frozen=True)
class Copy:
"""The columns of one open role the rule reads. `posted_at` is an ISO
string or None; `has_description` is whether `description` is non-empty."""
uid: str
company_key: str
title_key: str
country: str
place: str
host: str
has_description: bool
posted_at: str | None
def _country(c: Copy) -> str:
"""The row's country, or the one its place names when the column is empty
or `EU`, a board's word for several offices: "Netherlands" is NL."""
return c.country if c.country not in ("", "EU") else country_of(c.place)
def _tier(c: Copy) -> int:
"""How precisely a row says where: 0 names a town, 1 only a country, 2
neither."""
return 0 if places.towns(c.place) else 1 if _country(c) else 2
def same_posting(a: Copy, b: Copy) -> bool:
"""Whether two rows are one posting: a non-empty shared `company_key`,
the same `title_key`, compatible countries (equal, or either empty) and
towns that share a member or are empty on either side. Host is ignored."""
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
return False
ca, cb = _country(a), _country(b)
if ca and cb and ca != cb:
return False
ta, tb = places.towns(a.place), places.towns(b.place)
return not ta or not tb or bool(ta & tb)
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
"""The sort key that picks the kept copy of a group, smallest first: a
board over a search provider, then one with a description, then the
earliest `posted_at` (none last), then the smallest `uid`, so the choice
is the same on every run."""
return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at,
c.posted_at or "", c.uid)
def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
"""`uid -> dup_of` for every row: the uid of its group's kept copy, None
for the kept copy itself and for a row with no copies.
Rows are partitioned by `(company_key, title_key)`, a row with no
`company_key` standing alone. Within a partition rows are taken sharpest
tier first and in `keep_order`, and a row joins a group only when it is `same_posting` with every member.
Joining on one match would chain: a row with no country matches Japan and
Malaysia both and would make them one. A row naming a town that fits two
groups joins the first. A vaguer row joins the first group of its own tier
it fits, else the one sharper group it fits, else starts its own: with no
place it fits Belgium and the Netherlands, and joins neither. Each group keeps its `keep_order`
minimum, so a board with no place still wins over a search with a town."""
result: dict[str, str | None] = {}
parts: dict[tuple[str, str], list[Copy]] = {}
for row in rows:
result[row.uid] = None
if row.company_key:
parts.setdefault((row.company_key, row.title_key), []).append(row)
for part in parts.values():
groups: list[list[Copy]] = []
for row in sorted(part, key=lambda r: (_tier(r), keep_order(r))):
fits = [g for g in groups if all(same_posting(row, m) for m in g)]
level = [g for g in fits if _tier(g[0]) == _tier(row)]
if level or len(fits) == 1:
(level or fits)[0].append(row)
else:
groups.append([row])
for group in groups:
keeper = min(group, key=keep_order)
for row in group:
if row is not keeper:
result[row.uid] = keeper.uid
return result
+14 -1
View File
@@ -35,7 +35,7 @@ from functools import lru_cache
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
"label", "match_sql", "matches", "parse", "register", "suggest"]
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
def fold(text: str) -> str:
@@ -578,6 +578,19 @@ def _towns_in(folded: str) -> list[str]:
return list(dict.fromkeys(found))
def towns(place: str) -> frozenset[str]:
"""The towns a line names, each as its group's key, so `København K` and
`Copenhagen, Denmark` are both `{"copenhagen"}`: the first comma part of each
place, and every known town the line holds as whole words, so a list such as
`Aalborg, Copenhagen, Kolding` gives all three. A state or province after
the town is not a town, so `Mountain View, California` and `San Francisco,
California` share none. Empty for a line that names only a country, a region
or how the role is worked."""
found = {_group(fold(town)) for town in _towns(place)}
found.update(_towns_in(fold(place or "")))
return frozenset(found)
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
"""Open roles per region, over folded lines. A line is cut into every run of
words once, and each region asks whether it holds one of its phrases."""
+4 -12
View File
@@ -17,7 +17,9 @@ The same-host groups are the larger problem. SAP lists one job under 3 to 9 requ
## Decisions
**Two open roles are one posting when company, title, country and a town agree.** Same non-empty `company_key`, same `title_key`, same `country` (or either empty), and the sets of towns their `place` names share at least one member, where either side having no town counts as a match. Host does not matter, so a board's own repeats collapse like an aggregator's copy. The same title in Berlin and in Munich stays two roles, so a place filter still finds each.
**Two open roles are one posting when company, title, country and a town agree.** Same non-empty `company_key`, same `title_key`, the same country (the column, or the one the place names when the column is empty or `EU`), and town sets from `places.towns` that share a member, where either side having no town counts as a match. `places.towns` takes the first comma part of each place and every known town, so `Mountain View, California` and `San Francisco, California` share none. Host does not matter, so a board's own repeats collapse like an aggregator's copy. The same title in Berlin and in Munich stays two roles, so a place filter still finds each.
**A group is a set of rows that all match each other, never a chain.** A row with no place matches Belgium and the Netherlands, and a row with no country matches Japan and Malaysia; joining on one match merged both pairs on the live data. Rows are taken sharpest first (a town, then only a country, then neither) and join a group only when they match every member. A vaguer row joins a group of its own tier, else the one sharper group it fits, else starts its own.
**The group is stored, not computed per query.** `roles.dup_of` holds the uid of the group's kept role, NULL on the kept role and on a role without copies. The feed, the counts and the facets add `dup_of IS NULL`, which keeps `idx_roles_posted` usable; a window function over the filtered rows would scan them all on every page.
@@ -43,17 +45,7 @@ The storage and UI packages share files with packages E and G of `plan-feed-filt
|---|---|---|
| H, the rule | Sonnet | `app/dupes.py`, `tests/test_dupes.py` |
`app/dupes.py` is pure, with no store and no SQL:
```python
SEARCH_HOSTS: frozenset[str] # jobindex, adzuna, platsbanken
def towns(place: str, country: str) -> frozenset[str]
def same_posting(a: Copy, b: Copy) -> bool
def keep_order(c: Copy) -> tuple # sort key, smallest is kept
def assign(rows: Iterable[Copy]) -> dict[str, str | None] # uid -> dup_of
```
`Copy` is a frozen dataclass with `uid`, `company_key`, `title_key`, `country`, `place`, `host`, `has_description`, `posted_at`. `towns` splits on `,`, `;`, `|`, `/`, ` og `, ` och `, ` und `, ` and `, folds with `models._place_fold`, and drops country names (`models._COUNTRY_NAMES`), `Skåne`-style region words that follow a town, and words such as `hybrid`, `remote`, `position`, `mulighed for hjemmearbejde`; `København` and `Copenhagen` are one town. `assign` partitions rows by `(company_key, title_key)`, joins rows by `same_posting` transitively, and within each component points every row but the `keep_order` minimum at that minimum. The tests carry the five live cases above verbatim, Berlin against Munich, an empty place, an empty company and an empty country.
Done: `app/dupes.py` (`SEARCH_HOSTS`, `Copy`, `same_posting`, `keep_order`, `assign`) over `places.towns`, and `tests/test_dupes.py` with the live cases. On the live database it hides 1,080 of 33,881 open roles in 848 groups, 14 of them across hosts, in half a second. Known leftovers: Zeiss field technicians in `Remote - USA NH` and `Remote - USA WA` collapse, since a state code is not a town.
### Wave 2, after G of `plan-feed-filters.md` is merged, in parallel
+1 -1
View File
@@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1]
#: no database, so storage may use them. `main` assembles the app from
#: everything below it; `cli` and `__main__` are the entry points and may
#: reach anything.
LAYERS = ["models", "places", "seniority", "config", "storage", "jobs", "web", "main", "cli",
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli",
"__main__"]
APP = "app"
EXTERNAL = {"desk", "pipeline", "auth"}
+196
View File
@@ -0,0 +1,196 @@
"""`app.dupes` against postings as boards really wrote them. Pure: no store,
no network. The live cases are the 2026-09-29 database's."""
from __future__ import annotations
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
import pytest # noqa: E402
from app.dupes import Copy, assign, keep_order, same_posting # noqa: E402
from app.places import towns # noqa: E402
def cp(uid: str = "u", company: str = "acme", title: str = "engineer", country: str = "DE",
place: str = "", host: str = "greenhouse", desc: bool = True,
posted: str | None = None) -> Copy:
return Copy(uid, company, title, country, place, host, desc, posted)
#: `(company_key, title_key, country, host_a, place_a, host_b, place_b, kept host)`.
LIVE = [
("ericsson", "entry level 6g developer", "SE",
"platsbanken", "Lund", "eightfold", "Lund, Skåne, Sweden", "eightfold"),
("gn-group", "machine learning engineer", "DK",
"jobindex", "Ballerup", "workday", "Ballerup, Denmark", "workday"),
("kmd", "student worker finance", "DK",
"jobindex", "Ballerup, Ballerup, Aalborg, Aalborg", "successfactors", "Ballerup, Denmark",
"successfactors"),
("trackunit", "machine learning engineer", "DK",
"jobindex", "Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position",
"teamtailor", "København, Denmark; Aarhus, Denmark; Kolding, Denmark; Aalborg, Denmark",
"teamtailor"),
]
@pytest.mark.parametrize("company,title,country,ha,pa,hb,pb,kept", LIVE)
def test_live_cross_host_pairs(company, title, country, ha, pa, hb, pb, kept):
a = cp("a", company, title, country, pa, ha)
b = cp("b", company, title, country, pb, hb)
assert same_posting(a, b) and same_posting(b, a)
result = assign([a, b])
assert result == ({"a": "b", "b": None} if kept == hb else {"a": None, "b": "a"})
def test_sap_requisitions():
place = "Garching bei München (Munich), Germany"
title = "forward deployed chief ai engineering expert"
rows = [
cp("g3", "sap", title, "DE", place, "successfactors", False, "2026-09-02"),
cp("g1", "sap", title, "DE", place, "successfactors", False, "2026-09-01"),
cp("g2", "sap", title, "DE", place, "successfactors", False, "2026-09-01"),
cp("ny", "sap", title, "US", "New York, United States", "successfactors", True,
"2026-08-01"),
]
# The earliest posted_at, then the smallest uid, is kept; New York is alone
# although it is the only row with a description.
assert assign(rows) == {"g1": None, "g2": "g1", "g3": "g1", "ny": None}
def test_houston_state_code():
a = cp("a", "sap", "x", "US", "Houston, TX, United States")
b = cp("b", "sap", "x", "US", "Houston, United States")
assert same_posting(a, b)
def test_copenhagen_district_and_remote_phrase():
a = cp("a", "acme", "x", "DK", "København K og mulighed for hjemmearbejde")
b = cp("b", "acme", "x", "DK", "Copenhagen, Denmark")
assert towns(a.place) == {"copenhagen"}
assert same_posting(a, b)
def test_different_towns_are_not_the_same():
assert not same_posting(cp("a", place="Berlin, Germany"), cp("b", place="Munich, Germany"))
assert assign([cp("a", place="Berlin"), cp("b", place="Munich")]) == {"a": None, "b": None}
def test_empty_place_matches():
assert same_posting(cp("a", place=""), cp("b", place="Berlin"))
assert same_posting(cp("a", place=""), cp("b", place=""))
assert same_posting(cp("a", place="Germany"), cp("b", place="Berlin, Germany"))
def test_empty_company_never_grouped():
a, b = cp("a", company=""), cp("b", company="")
assert not same_posting(a, b)
assert assign([a, b]) == {"a": None, "b": None}
def test_other_company_or_title_is_not_the_same():
assert not same_posting(cp("a"), cp("b", company="other"))
assert not same_posting(cp("a"), cp("b", title="designer"))
def test_country_rules():
assert not same_posting(cp("a", country="DE", place="Berlin"),
cp("b", country="US", place="Berlin"))
assert same_posting(cp("a", country="", place="Berlin"),
cp("b", country="DE", place="Berlin, Germany"))
assert not same_posting(cp("a", country="", place="Berlin"),
cp("b", country="DE", place="Munich"))
@pytest.mark.parametrize("place,country,expected", [
("Lund, Skåne, Sweden", "SE", {"lund"}),
("Houston, TX, United States", "US", {"houston"}),
("New York, United States", "US", {"new york"}),
("Aarhus C", "DK", {"aarhus"}),
("Remote Berlin", "DE", {"berlin"}),
("Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", "DK",
{"aalborg", "copenhagen", "kolding", "aarhus"}),
("Berlin und Munich", "DE", {"berlin", "munich"}),
("Mountain View, California", "US", {"mountain view"}),
("Ningbo, Zhejiang, China", "CN", {"ningbo"}),
("Germany", "DE", set()),
("Remote", "", set()),
("", "", set()),
])
def test_towns(place, country, expected):
# A head `places` cannot parse, "remote berlin", may come along; it matches
# nothing, so only the towns that must be there and the empty case count.
found = towns(place)
assert found >= expected and (expected or not found)
def test_a_state_or_province_is_not_a_shared_town():
# Live false merges before `places.towns`: Databricks and Bosch.
assert not same_posting(cp("a", country="US", place="Mountain View, California"),
cp("b", country="US", place="San Francisco, California"))
assert not same_posting(cp("a", country="CN", place="Ningbo, Zhejiang, China"),
cp("b", country="CN", place="Hangzhou, Zhejiang, China"))
def test_assign_does_not_chain():
# A matches B by Berlin, B matches C by Hamburg, A and C do not match, so A
# cannot share C's group.
a = cp("a", place="Berlin", posted="2026-09-03")
b = cp("b", place="Berlin, Hamburg", posted="2026-09-02")
c = cp("c", place="Hamburg", posted="2026-09-01")
assert not same_posting(a, c)
assert assign([a, b, c]) == {"a": None, "b": "c", "c": None}
def test_assign_returns_every_uid():
rows = [cp("a"), cp("b", title="other"), cp("c", company="")]
assert assign(rows) == {"a": None, "b": None, "c": None}
assert assign([]) == {}
def test_assign_separate_groups_in_one_partition():
rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="Munich"),
cp("d", place="Munich")]
assert assign(rows) == {"a": None, "b": "a", "c": None, "d": "c"}
def test_keep_order_ranks():
board = cp("z", host="workday", desc=False, posted="2026-09-09")
search = cp("a", host="jobindex", desc=True, posted="2026-01-01")
assert min([search, board], key=keep_order) is board
with_desc = cp("z", desc=True, posted="2026-09-09")
without = cp("a", desc=False, posted="2026-01-01")
assert min([without, with_desc], key=keep_order) is with_desc
early = cp("z", posted="2026-01-01")
late = cp("a", posted="2026-02-01")
undated = cp("0", posted=None)
assert sorted([undated, late, early], key=keep_order) == [early, late, undated]
assert min([cp("b", posted="2026-01-01"), cp("a", posted="2026-01-01")],
key=keep_order).uid == "a"
def test_a_row_without_a_town_joins_only_one_group():
# "Germany" matches Berlin and Munich; joining both would make them one role.
rows = [cp("a", place="Berlin"), cp("b", place="Munich"), cp("c", place="Germany")]
assert assign(rows) == {"a": None, "b": None, "c": None}
rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="")]
assert assign(rows) == {"a": None, "b": "a", "c": "a"}
rows = [cp("a", place=""), cp("b", place="Germany")]
assert assign(rows) == {"a": None, "b": "a"}
def test_a_row_without_a_place_never_bridges_two_countries():
# Live: LEGO "Sales Associate", three rows with no place, one Belgium, one
# Netherlands, `country` empty on all five until the place names one.
rows = [cp("a", country="", place=""), cp("b", country="", place=""),
cp("c", country="", place=""), cp("d", country="BE", place="Belgium"),
cp("e", country="NL", place="Netherlands")]
got = assign(rows)
assert got["d"] is None and got["e"] is None
assert got == {"a": None, "b": "a", "c": "a", "d": None, "e": None}
assert not same_posting(cp("a", country="EU", place="Belgium"),
cp("b", country="EU", place="Netherlands"))