Places as a gazetteer, role_places, teaser and three seniority sources
Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
bbd7522163
10 files changed
+1437
-98
No files matched your search
+313
-5
@@ -19,8 +19,13 @@ several hundred milliseconds a query, as a set intersection on the words of a
|
||||
line it costs a few. `register` gives a connection that answer as the SQL
|
||||
function `place_match`, and `match_sql` returns the clause that calls it.
|
||||
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
|
||||
chip's text back from the address. This module opens no database; the caller
|
||||
hands over the places it holds, or its connection.
|
||||
chip's text back from the address.
|
||||
|
||||
The filter and the gates no longer match words: a role's places are resolved
|
||||
against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
|
||||
and a token is the places under the place it names. `matches` stays the rule
|
||||
for a `place:` token the gazetteer does not know. This module opens no
|
||||
database; the caller hands over the places it holds, or its connection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -32,10 +37,12 @@ from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
|
||||
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
|
||||
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
|
||||
country_of)
|
||||
|
||||
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
|
||||
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
|
||||
__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed",
|
||||
"Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse",
|
||||
"placeless", "register", "seed", "suggest", "towns"]
|
||||
|
||||
|
||||
def fold(text: str) -> str:
|
||||
@@ -538,6 +545,9 @@ _PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
|
||||
#: Danish boards append "and the possibility of working from home".
|
||||
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
|
||||
|
||||
#: A slash between two places, "Malmö/Lund", "Berlin / Munich".
|
||||
_SLASH = re.compile(r"\s*/\s*")
|
||||
|
||||
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
|
||||
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
|
||||
|
||||
@@ -665,3 +675,301 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
|
||||
regions.sort(key=lambda s: (-s.count, s.label))
|
||||
towns.sort(key=lambda s: (-s.count, s.label))
|
||||
return (countries + regions + towns)[:limit]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ gazetteer
|
||||
|
||||
#: The kinds of place, widest first: the index is how specific a kind is.
|
||||
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
|
||||
|
||||
#: Entries of `_CITIES` that name a region rather than a town.
|
||||
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
|
||||
"bay area"))
|
||||
|
||||
#: Words a place text holds when it names how or where-ever a role is worked
|
||||
#: rather than a place: "Remote", "Multiple Locations", "Home based". A text
|
||||
#: of these alone is no place, so it is never listed as unresolved.
|
||||
_PLACELESS = frozenset((
|
||||
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
|
||||
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
|
||||
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
|
||||
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
|
||||
|
||||
|
||||
def placeless(text: str) -> bool:
|
||||
"""Whether a place text names no place at all: empty, or only numbers and
|
||||
words such as `Remote`, `Hybrid`, `2 Locations`."""
|
||||
return all(word in _PLACELESS or word.isdigit() for word in fold(text).split())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Seed:
|
||||
"""One place as the code knows it: `key` is unique within `kind` (an ISO
|
||||
code for a country, a region's key, a town's group), `parent` the
|
||||
`(kind, key)` of the place it lies in, `aliases` the folds it is written
|
||||
as."""
|
||||
|
||||
kind: str
|
||||
key: str
|
||||
name: str
|
||||
parent: tuple[str, str] | None
|
||||
aliases: tuple[str, ...]
|
||||
|
||||
|
||||
def _subdivisions() -> list[tuple[str, str, str]]:
|
||||
"""`(country, code, name)` of every US state and Canadian province."""
|
||||
found = []
|
||||
for line in _SUBDIVISIONS.split("\n"):
|
||||
if line:
|
||||
country, rest = line.split(" ", 1)
|
||||
for entry in rest.split("|"):
|
||||
name, code = entry.strip().rsplit(" ", 1)
|
||||
found.append((country, code, name))
|
||||
return found
|
||||
|
||||
|
||||
def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
|
||||
"""The gazetteer the dictionaries in code describe, parents before their
|
||||
children, and the inclusions beyond the tree (Øresund holds the Capital
|
||||
Region and Skåne, which lie in two countries).
|
||||
|
||||
Europe is a continent over the countries of `_EUROPE`, and is also what
|
||||
`EU`, `EMEA` and `European Union` name. Every country of `_COUNTRY_NAMES`
|
||||
by its names; the US states and Canadian provinces as regions; the
|
||||
regions of `REGIONS`; the towns of the alias groups, of the regions and
|
||||
of `_CITIES`, one place per group (`Copenhagen`, `København`,
|
||||
`Kopenhamn` are one), in their region when a region lists them and else
|
||||
in their country."""
|
||||
europe = ("continent", "europe")
|
||||
seeds: dict[tuple[str, str], Seed] = {}
|
||||
|
||||
def add(kind: str, key: str, name: str, parent: tuple[str, str] | None,
|
||||
aliases: Iterable[str]) -> None:
|
||||
folds = tuple(dict.fromkeys(a for a in aliases if a))
|
||||
if (kind, key) in seeds:
|
||||
old = seeds[(kind, key)]
|
||||
folds = tuple(dict.fromkeys((*old.aliases, *folds)))
|
||||
seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds)
|
||||
else:
|
||||
seeds[(kind, key)] = Seed(kind, key, name, parent, folds)
|
||||
|
||||
add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS)))
|
||||
for code, names in _COUNTRY_LABELS.items():
|
||||
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
|
||||
for country, code, name in _subdivisions():
|
||||
add("region", f"{country}-{code}", name, ("country", country), (fold(name),))
|
||||
for region in REGIONS.values():
|
||||
parent = ("country", region.country) if region.country else europe
|
||||
add("region", region.key, region.label, parent,
|
||||
(v for name in (region.key, region.label, *region.names)
|
||||
for v in _variants(fold(name))))
|
||||
for name in sorted(_CITY_REGIONS):
|
||||
add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,))
|
||||
|
||||
# Towns, one per group, in the region that lists them first.
|
||||
town_region: dict[str, str] = {}
|
||||
for region in REGIONS.values():
|
||||
for name in region.towns:
|
||||
town_region.setdefault(_group(fold(name)), region.key)
|
||||
folds: dict[str, dict[str, None]] = {}
|
||||
shown: dict[str, str] = {}
|
||||
|
||||
def add_town(folded: str, name: str) -> None:
|
||||
key = _group(folded)
|
||||
folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded)))
|
||||
shown.setdefault(key, name)
|
||||
|
||||
for spellings in _ALIAS_GROUPS:
|
||||
for spelling in spellings:
|
||||
add_town(fold(spelling), spellings[0])
|
||||
for region in REGIONS.values():
|
||||
for name in region.towns:
|
||||
add_town(fold(name), name)
|
||||
for city in _CITIES:
|
||||
if city not in _CITY_REGIONS and city not in _REGION_BY_NAME:
|
||||
add_town(city, _TOWN_LABELS.get(city) or city.title())
|
||||
for key, forms in folds.items():
|
||||
country = next((_CITIES[f] for f in forms if f in _CITIES), "")
|
||||
if key in town_region:
|
||||
parent = ("region", town_region[key])
|
||||
elif country:
|
||||
parent = ("country", country)
|
||||
else:
|
||||
raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES")
|
||||
add("city", key, shown[key], parent, forms)
|
||||
|
||||
includes = [(("region", region.key), ("region", part))
|
||||
for region in REGIONS.values() for part in region.includes]
|
||||
return list(seeds.values()), includes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Place:
|
||||
"""A row of `places`."""
|
||||
|
||||
id: int
|
||||
kind: str
|
||||
key: str
|
||||
name: str
|
||||
parent_id: int | None
|
||||
|
||||
|
||||
class Gazetteer:
|
||||
"""The places, their aliases and what lies in what, as the database holds
|
||||
them, read into memory: what resolves a role's place text to places and a
|
||||
filter token to the places under it. Built by the store from its tables,
|
||||
and rebuilt when they change."""
|
||||
|
||||
def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]],
|
||||
aliases: Iterable[tuple[str, int]],
|
||||
includes: Iterable[tuple[int, int]] = ()) -> None:
|
||||
self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows}
|
||||
self.by_key: dict[tuple[str, str], int] = {
|
||||
(p.kind, p.key): p.id for p in self.places.values()}
|
||||
by_fold: dict[str, list[int]] = {}
|
||||
for folded, place_id in aliases:
|
||||
by_fold.setdefault(folded, []).append(place_id)
|
||||
self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()}
|
||||
self._longest = max((len(f.split()) for f in self.aliases), default=1)
|
||||
self._up: dict[int, set[int]] = {}
|
||||
self._down: dict[int, set[int]] = {}
|
||||
for p in self.places.values():
|
||||
if p.parent_id is not None:
|
||||
self._link(p.parent_id, p.id)
|
||||
for whole, part in includes:
|
||||
self._link(whole, part)
|
||||
self._ancestors: dict[int, frozenset[int]] = {}
|
||||
self._resolved: dict[str, tuple[int, ...]] = {}
|
||||
|
||||
def _link(self, whole: int, part: int) -> None:
|
||||
self._up.setdefault(part, set()).add(whole)
|
||||
self._down.setdefault(whole, set()).add(part)
|
||||
|
||||
def ancestors(self, place_id: int) -> frozenset[int]:
|
||||
"""Every place `place_id` lies in, through parents and inclusions."""
|
||||
if (found := self._ancestors.get(place_id)) is None:
|
||||
seen: set[int] = set()
|
||||
todo = list(self._up.get(place_id, ()))
|
||||
while todo:
|
||||
up = todo.pop()
|
||||
if up not in seen:
|
||||
seen.add(up)
|
||||
todo += self._up.get(up, ())
|
||||
found = self._ancestors[place_id] = frozenset(seen)
|
||||
return found
|
||||
|
||||
def under(self, ids: Iterable[int]) -> frozenset[int]:
|
||||
"""The places and every place that lies in one of them."""
|
||||
seen: set[int] = set()
|
||||
todo = list(ids)
|
||||
while todo:
|
||||
place_id = todo.pop()
|
||||
if place_id not in seen:
|
||||
seen.add(place_id)
|
||||
todo += self._down.get(place_id, ())
|
||||
return frozenset(seen)
|
||||
|
||||
def country(self, place_id: int) -> str:
|
||||
"""The ISO code of the country a place is in, '' above or outside one."""
|
||||
place: Place | None = self.places.get(place_id)
|
||||
while place is not None and place.kind != "country":
|
||||
place = self.places.get(place.parent_id) if place.parent_id is not None else None
|
||||
return place.key if place is not None else ""
|
||||
|
||||
def codes(self, ids: Iterable[int]) -> frozenset[str]:
|
||||
"""The ISO codes of the countries among `ids`, and `EU` with Europe:
|
||||
what a `roles.country` column holds for a role of one of them."""
|
||||
found = set()
|
||||
for place_id in ids:
|
||||
place = self.places[place_id]
|
||||
if place.kind == "country":
|
||||
found.add(place.key)
|
||||
elif (place.kind, place.key) == ("continent", "europe"):
|
||||
found.add("EU")
|
||||
return frozenset(found)
|
||||
|
||||
def _specific(self, place_id: int) -> int:
|
||||
return KINDS.index(self.places[place_id].kind)
|
||||
|
||||
def _hits(self, folded: str) -> list[tuple[int, ...]]:
|
||||
words = folded.split()
|
||||
hits = []
|
||||
i = 0
|
||||
while i < len(words):
|
||||
for n in range(min(self._longest, len(words) - i), 0, -1):
|
||||
if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None:
|
||||
hits.append(ids)
|
||||
i += n
|
||||
break
|
||||
else:
|
||||
i += 1
|
||||
return hits
|
||||
|
||||
def resolve(self, text: str) -> tuple[int, ...]:
|
||||
"""The places one place text names, most specific first from the left:
|
||||
`Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund`
|
||||
is Malmö and Lund, `Paderborn, Germany` is Germany, `Remote` is none.
|
||||
|
||||
Every alias the text holds as whole words is a candidate, longest
|
||||
first. A fold that names several places is the one in the country
|
||||
`country_of` reads off the text, if one is, and else the most specific
|
||||
(`Luxembourg` the town before the country). A candidate that another
|
||||
one lies in is dropped, and so is one outside the one country the text
|
||||
names, or, naming none, outside the regions it names in one country:
|
||||
`London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the
|
||||
Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several
|
||||
countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country
|
||||
`country_of` reads (from a state code, `Paris, TX`, or a code in
|
||||
parentheses, `Remote (DE)`), the text is that country."""
|
||||
folded = fold(text)
|
||||
if (found := self._resolved.get(folded)) is None:
|
||||
parts = _SLASH.split(_AND_HOME.sub("", text or ""))
|
||||
found = tuple(dict.fromkeys(
|
||||
p for part in parts for sub in _PART.split(part) for p in self._resolve(sub)))
|
||||
self._resolved[folded] = found
|
||||
return found
|
||||
|
||||
def _resolve(self, text: str) -> list[int]:
|
||||
"""`resolve` for one part of a text, cut where `or`, `/` or `;` end one
|
||||
place."""
|
||||
code = country_of(text)
|
||||
country = self.by_key.get(("country", code)) if code and code != "EU" else None
|
||||
within = self.under((country,)) if country is not None else frozenset()
|
||||
picked: list[int] = []
|
||||
for hit in self._hits(fold(text)):
|
||||
choice = [p for p in hit if p in within] or list(hit)
|
||||
best = max(choice, key=self._specific)
|
||||
if best not in picked:
|
||||
picked.append(best)
|
||||
# The widest units named hold the rest: one country, else the regions
|
||||
# of one country. Several countries are a list, not a path.
|
||||
for kind in ("country", "region"):
|
||||
named = [p for p in picked if self.places[p].kind == kind]
|
||||
if len({self.country(p) for p in named}) == 1:
|
||||
holds = self.under(named)
|
||||
picked = [p for p in picked if p in holds or self.places[p].kind == "continent"
|
||||
or any(p in self.ancestors(q) for q in named)]
|
||||
if named:
|
||||
break
|
||||
wider = {a for p in picked for a in self.ancestors(p)}
|
||||
picked = [p for p in picked if p not in wider]
|
||||
if country is not None and not any(p in within or p in self.ancestors(country)
|
||||
for p in picked):
|
||||
picked = [country]
|
||||
return picked
|
||||
|
||||
def lookup(self, token: str) -> frozenset[int] | None:
|
||||
"""The places a filter token names, before what lies in them: `country:DK`
|
||||
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
|
||||
every place written so (a town, or a region or a country by name).
|
||||
None when the gazetteer knows no such place."""
|
||||
tok = parse(token)
|
||||
if tok.kind == "country":
|
||||
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
|
||||
found = {self.by_key[key]} if key in self.by_key else set()
|
||||
elif tok.kind == "region":
|
||||
found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \
|
||||
else set(self.aliases.get(tok.value, ()))
|
||||
else:
|
||||
found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())}
|
||||
return frozenset(found) or None
|
||||
Reference in new issue
Block a user