Merge branch 'wp-h-dupes'
This commit is contained in:
commit
93bdad72a2
5 files changed
+335
-14
No files matched your search
+120
@@ -0,0 +1,120 @@
|
||||
"""Which open roles are one posting, and which copy is kept.
|
||||
|
||||
One job is often listed more than once: by a search provider and by the
|
||||
company's own board, or by one board under several requisition ids (SAP lists
|
||||
one job under 3 to 9 of them). The stored cluster key `(company_key,
|
||||
title_key, place_key)` finds none of the cross-host pairs, because the places
|
||||
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
|
||||
`Lund, Skåne, Sweden`. So the rule here compares towns as `places.towns` reads
|
||||
them, not place strings.
|
||||
|
||||
Two rows are one posting when they have the same non-empty `company_key`, the
|
||||
same `title_key`, the same `country` (or either has none), and the sets of
|
||||
towns their places name share a member, where a row with no town matches
|
||||
every town. Host does not matter. The same title in Berlin and in Munich
|
||||
stays two roles, so a place filter still finds each. A group is a set of
|
||||
rows that each match all the others, never a chain: a row with no place
|
||||
matches Belgium and the Netherlands, and must not make them one role.
|
||||
|
||||
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
|
||||
it. Pure by design: no store, no SQL, no network, so the rule is tested on
|
||||
the strings boards really write and the storage layer only feeds it rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app import places
|
||||
from app.models import country_of
|
||||
|
||||
#: Hosts that search across companies rather than run a company's own board.
|
||||
#: A board closes a role reliably and carries the full text, so it is kept
|
||||
#: over a copy found here.
|
||||
SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Copy:
|
||||
"""The columns of one open role the rule reads. `posted_at` is an ISO
|
||||
string or None; `has_description` is whether `description` is non-empty."""
|
||||
|
||||
uid: str
|
||||
company_key: str
|
||||
title_key: str
|
||||
country: str
|
||||
place: str
|
||||
host: str
|
||||
has_description: bool
|
||||
posted_at: str | None
|
||||
|
||||
|
||||
def _country(c: Copy) -> str:
|
||||
"""The row's country, or the one its place names when the column is empty
|
||||
or `EU`, a board's word for several offices: "Netherlands" is NL."""
|
||||
return c.country if c.country not in ("", "EU") else country_of(c.place)
|
||||
|
||||
|
||||
def _tier(c: Copy) -> int:
|
||||
"""How precisely a row says where: 0 names a town, 1 only a country, 2
|
||||
neither."""
|
||||
return 0 if places.towns(c.place) else 1 if _country(c) else 2
|
||||
|
||||
|
||||
def same_posting(a: Copy, b: Copy) -> bool:
|
||||
"""Whether two rows are one posting: a non-empty shared `company_key`,
|
||||
the same `title_key`, compatible countries (equal, or either empty) and
|
||||
towns that share a member or are empty on either side. Host is ignored."""
|
||||
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
|
||||
return False
|
||||
ca, cb = _country(a), _country(b)
|
||||
if ca and cb and ca != cb:
|
||||
return False
|
||||
ta, tb = places.towns(a.place), places.towns(b.place)
|
||||
return not ta or not tb or bool(ta & tb)
|
||||
|
||||
|
||||
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
||||
"""The sort key that picks the kept copy of a group, smallest first: a
|
||||
board over a search provider, then one with a description, then the
|
||||
earliest `posted_at` (none last), then the smallest `uid`, so the choice
|
||||
is the same on every run."""
|
||||
return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at,
|
||||
c.posted_at or "", c.uid)
|
||||
|
||||
|
||||
def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
|
||||
"""`uid -> dup_of` for every row: the uid of its group's kept copy, None
|
||||
for the kept copy itself and for a row with no copies.
|
||||
|
||||
Rows are partitioned by `(company_key, title_key)`, a row with no
|
||||
`company_key` standing alone. Within a partition rows are taken sharpest
|
||||
tier first and in `keep_order`, and a row joins a group only when it is `same_posting` with every member.
|
||||
Joining on one match would chain: a row with no country matches Japan and
|
||||
Malaysia both and would make them one. A row naming a town that fits two
|
||||
groups joins the first. A vaguer row joins the first group of its own tier
|
||||
it fits, else the one sharper group it fits, else starts its own: with no
|
||||
place it fits Belgium and the Netherlands, and joins neither. Each group keeps its `keep_order`
|
||||
minimum, so a board with no place still wins over a search with a town."""
|
||||
result: dict[str, str | None] = {}
|
||||
parts: dict[tuple[str, str], list[Copy]] = {}
|
||||
for row in rows:
|
||||
result[row.uid] = None
|
||||
if row.company_key:
|
||||
parts.setdefault((row.company_key, row.title_key), []).append(row)
|
||||
for part in parts.values():
|
||||
groups: list[list[Copy]] = []
|
||||
for row in sorted(part, key=lambda r: (_tier(r), keep_order(r))):
|
||||
fits = [g for g in groups if all(same_posting(row, m) for m in g)]
|
||||
level = [g for g in fits if _tier(g[0]) == _tier(row)]
|
||||
if level or len(fits) == 1:
|
||||
(level or fits)[0].append(row)
|
||||
else:
|
||||
groups.append([row])
|
||||
for group in groups:
|
||||
keeper = min(group, key=keep_order)
|
||||
for row in group:
|
||||
if row is not keeper:
|
||||
result[row.uid] = keeper.uid
|
||||
return result
|
||||
+14
-1
@@ -35,7 +35,7 @@ from functools import lru_cache
|
||||
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
|
||||
|
||||
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
|
||||
"label", "match_sql", "matches", "parse", "register", "suggest"]
|
||||
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
|
||||
|
||||
|
||||
def fold(text: str) -> str:
|
||||
@@ -578,6 +578,19 @@ def _towns_in(folded: str) -> list[str]:
|
||||
return list(dict.fromkeys(found))
|
||||
|
||||
|
||||
def towns(place: str) -> frozenset[str]:
|
||||
"""The towns a line names, each as its group's key, so `København K` and
|
||||
`Copenhagen, Denmark` are both `{"copenhagen"}`: the first comma part of each
|
||||
place, and every known town the line holds as whole words, so a list such as
|
||||
`Aalborg, Copenhagen, Kolding` gives all three. A state or province after
|
||||
the town is not a town, so `Mountain View, California` and `San Francisco,
|
||||
California` share none. Empty for a line that names only a country, a region
|
||||
or how the role is worked."""
|
||||
found = {_group(fold(town)) for town in _towns(place)}
|
||||
found.update(_towns_in(fold(place or "")))
|
||||
return frozenset(found)
|
||||
|
||||
|
||||
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
|
||||
"""Open roles per region, over folded lines. A line is cut into every run of
|
||||
words once, and each region asks whether it holds one of its phrases."""
|
||||
|
||||
+4
-12
@@ -17,7 +17,9 @@ The same-host groups are the larger problem. SAP lists one job under 3 to 9 requ
|
||||
|
||||
## Decisions
|
||||
|
||||
**Two open roles are one posting when company, title, country and a town agree.** Same non-empty `company_key`, same `title_key`, same `country` (or either empty), and the sets of towns their `place` names share at least one member, where either side having no town counts as a match. Host does not matter, so a board's own repeats collapse like an aggregator's copy. The same title in Berlin and in Munich stays two roles, so a place filter still finds each.
|
||||
**Two open roles are one posting when company, title, country and a town agree.** Same non-empty `company_key`, same `title_key`, the same country (the column, or the one the place names when the column is empty or `EU`), and town sets from `places.towns` that share a member, where either side having no town counts as a match. `places.towns` takes the first comma part of each place and every known town, so `Mountain View, California` and `San Francisco, California` share none. Host does not matter, so a board's own repeats collapse like an aggregator's copy. The same title in Berlin and in Munich stays two roles, so a place filter still finds each.
|
||||
|
||||
**A group is a set of rows that all match each other, never a chain.** A row with no place matches Belgium and the Netherlands, and a row with no country matches Japan and Malaysia; joining on one match merged both pairs on the live data. Rows are taken sharpest first (a town, then only a country, then neither) and join a group only when they match every member. A vaguer row joins a group of its own tier, else the one sharper group it fits, else starts its own.
|
||||
|
||||
**The group is stored, not computed per query.** `roles.dup_of` holds the uid of the group's kept role, NULL on the kept role and on a role without copies. The feed, the counts and the facets add `dup_of IS NULL`, which keeps `idx_roles_posted` usable; a window function over the filtered rows would scan them all on every page.
|
||||
|
||||
@@ -43,17 +45,7 @@ The storage and UI packages share files with packages E and G of `plan-feed-filt
|
||||
|---|---|---|
|
||||
| H, the rule | Sonnet | `app/dupes.py`, `tests/test_dupes.py` |
|
||||
|
||||
`app/dupes.py` is pure, with no store and no SQL:
|
||||
|
||||
```python
|
||||
SEARCH_HOSTS: frozenset[str] # jobindex, adzuna, platsbanken
|
||||
def towns(place: str, country: str) -> frozenset[str]
|
||||
def same_posting(a: Copy, b: Copy) -> bool
|
||||
def keep_order(c: Copy) -> tuple # sort key, smallest is kept
|
||||
def assign(rows: Iterable[Copy]) -> dict[str, str | None] # uid -> dup_of
|
||||
```
|
||||
|
||||
`Copy` is a frozen dataclass with `uid`, `company_key`, `title_key`, `country`, `place`, `host`, `has_description`, `posted_at`. `towns` splits on `,`, `;`, `|`, `/`, ` og `, ` och `, ` und `, ` and `, folds with `models._place_fold`, and drops country names (`models._COUNTRY_NAMES`), `Skåne`-style region words that follow a town, and words such as `hybrid`, `remote`, `position`, `mulighed for hjemmearbejde`; `København` and `Copenhagen` are one town. `assign` partitions rows by `(company_key, title_key)`, joins rows by `same_posting` transitively, and within each component points every row but the `keep_order` minimum at that minimum. The tests carry the five live cases above verbatim, Berlin against Munich, an empty place, an empty company and an empty country.
|
||||
Done: `app/dupes.py` (`SEARCH_HOSTS`, `Copy`, `same_posting`, `keep_order`, `assign`) over `places.towns`, and `tests/test_dupes.py` with the live cases. On the live database it hides 1,080 of 33,881 open roles in 848 groups, 14 of them across hosts, in half a second. Known leftovers: Zeiss field technicians in `Remote - USA NH` and `Remote - USA WA` collapse, since a state code is not a town.
|
||||
|
||||
### Wave 2, after G of `plan-feed-filters.md` is merged, in parallel
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
#: no database, so storage may use them. `main` assembles the app from
|
||||
#: everything below it; `cli` and `__main__` are the entry points and may
|
||||
#: reach anything.
|
||||
LAYERS = ["models", "places", "seniority", "config", "storage", "jobs", "web", "main", "cli",
|
||||
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli",
|
||||
"__main__"]
|
||||
APP = "app"
|
||||
EXTERNAL = {"desk", "pipeline", "auth"}
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""`app.dupes` against postings as boards really wrote them. Pure: no store,
|
||||
no network. The live cases are the 2026-09-29 database's."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
import pytest # noqa: E402
|
||||
|
||||
from app.dupes import Copy, assign, keep_order, same_posting # noqa: E402
|
||||
from app.places import towns # noqa: E402
|
||||
|
||||
|
||||
def cp(uid: str = "u", company: str = "acme", title: str = "engineer", country: str = "DE",
|
||||
place: str = "", host: str = "greenhouse", desc: bool = True,
|
||||
posted: str | None = None) -> Copy:
|
||||
return Copy(uid, company, title, country, place, host, desc, posted)
|
||||
|
||||
|
||||
#: `(company_key, title_key, country, host_a, place_a, host_b, place_b, kept host)`.
|
||||
LIVE = [
|
||||
("ericsson", "entry level 6g developer", "SE",
|
||||
"platsbanken", "Lund", "eightfold", "Lund, Skåne, Sweden", "eightfold"),
|
||||
("gn-group", "machine learning engineer", "DK",
|
||||
"jobindex", "Ballerup", "workday", "Ballerup, Denmark", "workday"),
|
||||
("kmd", "student worker finance", "DK",
|
||||
"jobindex", "Ballerup, Ballerup, Aalborg, Aalborg", "successfactors", "Ballerup, Denmark",
|
||||
"successfactors"),
|
||||
("trackunit", "machine learning engineer", "DK",
|
||||
"jobindex", "Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position",
|
||||
"teamtailor", "København, Denmark; Aarhus, Denmark; Kolding, Denmark; Aalborg, Denmark",
|
||||
"teamtailor"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("company,title,country,ha,pa,hb,pb,kept", LIVE)
|
||||
def test_live_cross_host_pairs(company, title, country, ha, pa, hb, pb, kept):
|
||||
a = cp("a", company, title, country, pa, ha)
|
||||
b = cp("b", company, title, country, pb, hb)
|
||||
assert same_posting(a, b) and same_posting(b, a)
|
||||
result = assign([a, b])
|
||||
assert result == ({"a": "b", "b": None} if kept == hb else {"a": None, "b": "a"})
|
||||
|
||||
|
||||
def test_sap_requisitions():
|
||||
place = "Garching bei München (Munich), Germany"
|
||||
title = "forward deployed chief ai engineering expert"
|
||||
rows = [
|
||||
cp("g3", "sap", title, "DE", place, "successfactors", False, "2026-09-02"),
|
||||
cp("g1", "sap", title, "DE", place, "successfactors", False, "2026-09-01"),
|
||||
cp("g2", "sap", title, "DE", place, "successfactors", False, "2026-09-01"),
|
||||
cp("ny", "sap", title, "US", "New York, United States", "successfactors", True,
|
||||
"2026-08-01"),
|
||||
]
|
||||
# The earliest posted_at, then the smallest uid, is kept; New York is alone
|
||||
# although it is the only row with a description.
|
||||
assert assign(rows) == {"g1": None, "g2": "g1", "g3": "g1", "ny": None}
|
||||
|
||||
|
||||
def test_houston_state_code():
|
||||
a = cp("a", "sap", "x", "US", "Houston, TX, United States")
|
||||
b = cp("b", "sap", "x", "US", "Houston, United States")
|
||||
assert same_posting(a, b)
|
||||
|
||||
|
||||
def test_copenhagen_district_and_remote_phrase():
|
||||
a = cp("a", "acme", "x", "DK", "København K og mulighed for hjemmearbejde")
|
||||
b = cp("b", "acme", "x", "DK", "Copenhagen, Denmark")
|
||||
assert towns(a.place) == {"copenhagen"}
|
||||
assert same_posting(a, b)
|
||||
|
||||
|
||||
def test_different_towns_are_not_the_same():
|
||||
assert not same_posting(cp("a", place="Berlin, Germany"), cp("b", place="Munich, Germany"))
|
||||
assert assign([cp("a", place="Berlin"), cp("b", place="Munich")]) == {"a": None, "b": None}
|
||||
|
||||
|
||||
def test_empty_place_matches():
|
||||
assert same_posting(cp("a", place=""), cp("b", place="Berlin"))
|
||||
assert same_posting(cp("a", place=""), cp("b", place=""))
|
||||
assert same_posting(cp("a", place="Germany"), cp("b", place="Berlin, Germany"))
|
||||
|
||||
|
||||
def test_empty_company_never_grouped():
|
||||
a, b = cp("a", company=""), cp("b", company="")
|
||||
assert not same_posting(a, b)
|
||||
assert assign([a, b]) == {"a": None, "b": None}
|
||||
|
||||
|
||||
def test_other_company_or_title_is_not_the_same():
|
||||
assert not same_posting(cp("a"), cp("b", company="other"))
|
||||
assert not same_posting(cp("a"), cp("b", title="designer"))
|
||||
|
||||
|
||||
def test_country_rules():
|
||||
assert not same_posting(cp("a", country="DE", place="Berlin"),
|
||||
cp("b", country="US", place="Berlin"))
|
||||
assert same_posting(cp("a", country="", place="Berlin"),
|
||||
cp("b", country="DE", place="Berlin, Germany"))
|
||||
assert not same_posting(cp("a", country="", place="Berlin"),
|
||||
cp("b", country="DE", place="Munich"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("place,country,expected", [
|
||||
("Lund, Skåne, Sweden", "SE", {"lund"}),
|
||||
("Houston, TX, United States", "US", {"houston"}),
|
||||
("New York, United States", "US", {"new york"}),
|
||||
("Aarhus C", "DK", {"aarhus"}),
|
||||
("Remote Berlin", "DE", {"berlin"}),
|
||||
("Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position", "DK",
|
||||
{"aalborg", "copenhagen", "kolding", "aarhus"}),
|
||||
("Berlin und Munich", "DE", {"berlin", "munich"}),
|
||||
("Mountain View, California", "US", {"mountain view"}),
|
||||
("Ningbo, Zhejiang, China", "CN", {"ningbo"}),
|
||||
("Germany", "DE", set()),
|
||||
("Remote", "", set()),
|
||||
("", "", set()),
|
||||
])
|
||||
def test_towns(place, country, expected):
|
||||
# A head `places` cannot parse, "remote berlin", may come along; it matches
|
||||
# nothing, so only the towns that must be there and the empty case count.
|
||||
found = towns(place)
|
||||
assert found >= expected and (expected or not found)
|
||||
|
||||
|
||||
def test_a_state_or_province_is_not_a_shared_town():
|
||||
# Live false merges before `places.towns`: Databricks and Bosch.
|
||||
assert not same_posting(cp("a", country="US", place="Mountain View, California"),
|
||||
cp("b", country="US", place="San Francisco, California"))
|
||||
assert not same_posting(cp("a", country="CN", place="Ningbo, Zhejiang, China"),
|
||||
cp("b", country="CN", place="Hangzhou, Zhejiang, China"))
|
||||
|
||||
|
||||
def test_assign_does_not_chain():
|
||||
# A matches B by Berlin, B matches C by Hamburg, A and C do not match, so A
|
||||
# cannot share C's group.
|
||||
a = cp("a", place="Berlin", posted="2026-09-03")
|
||||
b = cp("b", place="Berlin, Hamburg", posted="2026-09-02")
|
||||
c = cp("c", place="Hamburg", posted="2026-09-01")
|
||||
assert not same_posting(a, c)
|
||||
assert assign([a, b, c]) == {"a": None, "b": "c", "c": None}
|
||||
|
||||
|
||||
def test_assign_returns_every_uid():
|
||||
rows = [cp("a"), cp("b", title="other"), cp("c", company="")]
|
||||
assert assign(rows) == {"a": None, "b": None, "c": None}
|
||||
assert assign([]) == {}
|
||||
|
||||
|
||||
def test_assign_separate_groups_in_one_partition():
|
||||
rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="Munich"),
|
||||
cp("d", place="Munich")]
|
||||
assert assign(rows) == {"a": None, "b": "a", "c": None, "d": "c"}
|
||||
|
||||
|
||||
def test_keep_order_ranks():
|
||||
board = cp("z", host="workday", desc=False, posted="2026-09-09")
|
||||
search = cp("a", host="jobindex", desc=True, posted="2026-01-01")
|
||||
assert min([search, board], key=keep_order) is board
|
||||
with_desc = cp("z", desc=True, posted="2026-09-09")
|
||||
without = cp("a", desc=False, posted="2026-01-01")
|
||||
assert min([without, with_desc], key=keep_order) is with_desc
|
||||
early = cp("z", posted="2026-01-01")
|
||||
late = cp("a", posted="2026-02-01")
|
||||
undated = cp("0", posted=None)
|
||||
assert sorted([undated, late, early], key=keep_order) == [early, late, undated]
|
||||
assert min([cp("b", posted="2026-01-01"), cp("a", posted="2026-01-01")],
|
||||
key=keep_order).uid == "a"
|
||||
|
||||
|
||||
def test_a_row_without_a_town_joins_only_one_group():
|
||||
# "Germany" matches Berlin and Munich; joining both would make them one role.
|
||||
rows = [cp("a", place="Berlin"), cp("b", place="Munich"), cp("c", place="Germany")]
|
||||
assert assign(rows) == {"a": None, "b": None, "c": None}
|
||||
rows = [cp("a", place="Berlin"), cp("b", place="Berlin"), cp("c", place="")]
|
||||
assert assign(rows) == {"a": None, "b": "a", "c": "a"}
|
||||
rows = [cp("a", place=""), cp("b", place="Germany")]
|
||||
assert assign(rows) == {"a": None, "b": "a"}
|
||||
|
||||
|
||||
def test_a_row_without_a_place_never_bridges_two_countries():
|
||||
# Live: LEGO "Sales Associate", three rows with no place, one Belgium, one
|
||||
# Netherlands, `country` empty on all five until the place names one.
|
||||
rows = [cp("a", country="", place=""), cp("b", country="", place=""),
|
||||
cp("c", country="", place=""), cp("d", country="BE", place="Belgium"),
|
||||
cp("e", country="NL", place="Netherlands")]
|
||||
got = assign(rows)
|
||||
assert got["d"] is None and got["e"] is None
|
||||
assert got == {"a": None, "b": "a", "c": "a", "d": None, "e": None}
|
||||
assert not same_posting(cp("a", country="EU", place="Belgium"),
|
||||
cp("b", country="EU", place="Netherlands"))
|
||||
Reference in new issue
Block a user