Provider names, the CLI's choices and the dupes rule's search hosts now come from readers.PROVIDERS, and the regions, work modes and remote words from the modules' REGIONS, MODES and REMOTE, read with getattr so main works before and after the readers merge. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
230 lines
9.6 KiB
Python
230 lines
9.6 KiB
Python
"""Which open roles are one posting, and which copy is kept.
|
|
|
|
One job is often listed more than once: by a search provider and by the
|
|
company's own board, or by one board under several requisition ids (SAP lists
|
|
one job under 3 to 9 of them). The stored cluster key `(company_key,
|
|
title_key, place_key)` finds none of the cross-host pairs, because the places
|
|
never agree exactly: `Ballerup` against `Ballerup, Denmark`, `Lund` against
|
|
`Lund, Skåne, Sweden`. So the rule here compares towns as `places.towns` reads
|
|
them, not place strings.
|
|
|
|
Two rows are one posting when they have the same non-empty `company_key`, the
|
|
same `title_key`, the same `country` (or either has none), and the sets of
|
|
towns their places name share a member, where a row with no town matches
|
|
every town. Host does not matter. The same title in Berlin and in Munich
|
|
stays two roles, so a place filter still finds each. A group is a set of
|
|
rows that each match all the others, never a chain: a row with no place
|
|
matches Belgium and the Netherlands, and must not make them one role.
|
|
|
|
Each group keeps one copy (`keep_order`, smallest wins) and the rest point at
|
|
it.
|
|
|
|
A second rule folds what the first leaves apart on purpose: one posting per
|
|
place. A host that lists one job once per office, each with its own id, gives
|
|
rows with the same company and title whose towns never meet. `fold` merges
|
|
those rows of one host into the kept role of the first group, when the texts
|
|
do not say they are different jobs (`similar`), and reports them in
|
|
`Folded.placed`: the store gives the kept role each such copy's place with the
|
|
copy's own link. This holds for every host, not only readers that declare
|
|
`READS.places == "copies"`: on the live database 1,739 company-and-title
|
|
groups (5,024 roles) are this, in 18 hosts, 1,700 of them within one host, and
|
|
Greenhouse, Ashby and SmartRecruiters carry most, not HiBob. Rows whose texts
|
|
share fewer than `SAME_TEXT` of their words, or without a link to keep, or
|
|
listed by two hosts, stay apart. Pure by design: no store, no SQL, no network, so the rule is tested on
|
|
the strings boards really write and the storage layer only feeds it rows.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import sys
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from typing import NamedTuple
|
|
|
|
from app import places
|
|
from app.models import country_of
|
|
|
|
|
|
def is_search_host(host: str) -> bool:
|
|
"""Whether `host` is a provider that searches across companies rather than
|
|
a company's own board: a reader of `scope="search"`. A board closes a role
|
|
reliably and carries the full text, so it is kept over a copy found here.
|
|
`plugins` is a top-level folder beside `app`, reached lazily, with the
|
|
repo root put on the path when it is not there yet, as
|
|
`app.models._declared_posted` does."""
|
|
try:
|
|
from plugins import readers
|
|
except ImportError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from plugins import readers
|
|
|
|
return host in readers.PROVIDERS
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Copy:
|
|
"""The columns of one open role the rule reads. `posted_at` is an ISO
|
|
string or None; `has_description` is whether `description` is non-empty."""
|
|
|
|
uid: str
|
|
company_key: str
|
|
title_key: str
|
|
country: str
|
|
place: str
|
|
host: str
|
|
has_description: bool
|
|
posted_at: str | None
|
|
#: The start of the description, else the teaser, for `similar`; '' when
|
|
#: the row has neither.
|
|
text: str = ""
|
|
#: The link that opens this row on its host.
|
|
url: str = ""
|
|
|
|
|
|
def _country(c: Copy) -> str:
|
|
"""The row's country, or the one its place names when the column is empty
|
|
or `EU`, a board's word for several offices: "Netherlands" is NL."""
|
|
return c.country if c.country not in ("", "EU") else country_of(c.place)
|
|
|
|
|
|
def _tier(c: Copy) -> int:
|
|
"""How precisely a row says where: 0 names a town, 1 only a country, 2
|
|
neither."""
|
|
return 0 if places.towns(c.place) else 1 if _country(c) else 2
|
|
|
|
|
|
def same_posting(a: Copy, b: Copy) -> bool:
|
|
"""Whether two rows are one posting: a non-empty shared `company_key`,
|
|
the same `title_key`, compatible countries (equal, or either empty) and
|
|
towns that share a member or are empty on either side. Host is ignored."""
|
|
if not a.company_key or a.company_key != b.company_key or a.title_key != b.title_key:
|
|
return False
|
|
ca, cb = _country(a), _country(b)
|
|
if ca and cb and ca != cb:
|
|
return False
|
|
ta, tb = places.towns(a.place), places.towns(b.place)
|
|
return not ta or not tb or bool(ta & tb)
|
|
|
|
|
|
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
|
"""The sort key that picks the kept copy of a group, smallest first: a
|
|
board over a search provider, then one with a description, then the
|
|
earliest `posted_at` (none last), then the smallest `uid`, so the choice
|
|
is the same on every run."""
|
|
return (is_search_host(c.host), not c.has_description, not c.posted_at,
|
|
c.posted_at or "", c.uid)
|
|
|
|
|
|
def assign(rows: Iterable[Copy]) -> dict[str, str | None]:
|
|
"""`uid -> dup_of` for every row: the uid of its group's kept copy, None
|
|
for the kept copy itself and for a row with no copies.
|
|
|
|
Rows are partitioned by `(company_key, title_key)`, a row with no
|
|
`company_key` standing alone. Within a partition rows are taken sharpest
|
|
tier first and in `keep_order`, and a row joins a group only when it is `same_posting` with every member.
|
|
Joining on one match would chain: a row with no country matches Japan and
|
|
Malaysia both and would make them one. A row naming a town that fits two
|
|
groups joins the first. A vaguer row joins the first group of its own tier
|
|
it fits, else the one sharper group it fits, else starts its own: with no
|
|
place it fits Belgium and the Netherlands, and joins neither. Each group keeps its `keep_order`
|
|
minimum, so a board with no place still wins over a search with a town."""
|
|
result: dict[str, str | None] = {}
|
|
parts: dict[tuple[str, str], list[Copy]] = {}
|
|
for row in rows:
|
|
result[row.uid] = None
|
|
if row.company_key:
|
|
parts.setdefault((row.company_key, row.title_key), []).append(row)
|
|
for part in parts.values():
|
|
groups: list[list[Copy]] = []
|
|
for row in sorted(part, key=lambda r: (_tier(r), keep_order(r))):
|
|
fits = [g for g in groups if all(same_posting(row, m) for m in g)]
|
|
level = [g for g in fits if _tier(g[0]) == _tier(row)]
|
|
if level or len(fits) == 1:
|
|
(level or fits)[0].append(row)
|
|
else:
|
|
groups.append([row])
|
|
for group in groups:
|
|
keeper = min(group, key=keep_order)
|
|
for row in group:
|
|
if row is not keeper:
|
|
result[row.uid] = keeper.uid
|
|
return result
|
|
|
|
|
|
#: The share of words two texts must have in common (Jaccard, over the first
|
|
#: 3,000 characters) for one job in two places. Measured on the live
|
|
#: database: copies of a posting share nearly all of it, two jobs that
|
|
#: happen to share a title share little more than boilerplate, and a
|
|
#: translation shares none, which keeps it apart.
|
|
SAME_TEXT = 0.3
|
|
|
|
TEXT_CHARS = 3000
|
|
|
|
|
|
@lru_cache(maxsize=8192)
|
|
def _words(text: str) -> frozenset[str]:
|
|
return frozenset(re.findall(r"\w{3,}", text[:TEXT_CHARS].lower()))
|
|
|
|
|
|
def similar(a: str, b: str) -> bool:
|
|
"""Whether two texts can be one job: a text that is missing says nothing
|
|
against it, else they share at least `SAME_TEXT` of their words."""
|
|
wa, wb = _words(a), _words(b)
|
|
if not wa or not wb:
|
|
return True
|
|
return len(wa & wb) / len(wa | wb) >= SAME_TEXT
|
|
|
|
|
|
class Folded(NamedTuple):
|
|
"""`assign`'s answer plus the rows that are one place of another role:
|
|
`placed` holds the uid of each copy that `fold` merged by place."""
|
|
|
|
dup_of: dict[str, str | None]
|
|
placed: frozenset[str]
|
|
|
|
|
|
def fold(rows: Iterable[Copy]) -> Folded:
|
|
"""`assign`, then the per-place copies of each host folded in.
|
|
|
|
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
|
again, in `keep_order`: a row joins the first group whose members are all
|
|
of its host and `similar` to it. Only a row with a link can be folded, as
|
|
its link is what the kept role keeps. Every row of a merged group points
|
|
at its first row, the rows that pointed at a folded one included."""
|
|
rows = list(rows)
|
|
result = assign(rows)
|
|
by_uid = {r.uid: r for r in rows}
|
|
parts: dict[tuple[str, str], list[Copy]] = {}
|
|
for uid, dup in result.items():
|
|
row = by_uid[uid]
|
|
if dup is None and row.company_key and row.url:
|
|
parts.setdefault((row.company_key, row.title_key), []).append(row)
|
|
moved: dict[str, str] = {}
|
|
for part in parts.values():
|
|
groups: list[list[Copy]] = []
|
|
for row in sorted(part, key=keep_order):
|
|
for group in groups:
|
|
if all(m.host == row.host and similar(m.text, row.text) for m in group):
|
|
group.append(row)
|
|
break
|
|
else:
|
|
groups.append([row])
|
|
for keeper, *others in groups:
|
|
moved.update((other.uid, keeper.uid) for other in others)
|
|
for uid, dup in result.items():
|
|
if uid in moved:
|
|
result[uid] = moved[uid]
|
|
elif dup in moved:
|
|
result[uid] = moved[dup]
|
|
return Folded(result, frozenset(moved))
|
|
|
|
|
|
def place_line(texts: Iterable[str]) -> str:
|
|
"""The line a feed row shows for a role's places: the first, and "+N" for
|
|
the N others. Repeated texts count once."""
|
|
found = list(dict.fromkeys(t.strip() for t in texts if t and t.strip()))
|
|
return f"{found[0]} +{len(found) - 1}" if len(found) > 1 else "".join(found)
|