Places as a gazetteer, role_places, teaser and three seniority sources

Migration 19 adds places, place_aliases, place_includes (seeded from the
dictionaries in app/models.py and app/places.py, Europe as a continent over
the European countries) and role_places, one row per place of a role,
resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them
from raw["places"], else the place line cut at ";"; the migration builds them
from each role's place column.

The Place filter and the country and place gates are joins over the tree:
country:DK is any place under Denmark, region:skane any town in Skane, a
country gate is met by any place in its countries, and the EU fallback of
the word matcher is no longer used. A place: token the gazetteer does not
know still matches whole words. Store gains gazetteer, role_places,
unresolved_places, add_place_alias, add_place and relink_places.

roles.teaser is kept-filled and embedded while there is no description; a
vector is dropped when a better text arrives. seniority_model, _stated and
_title sit behind roles.seniority, which stays the first filled of them.

TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads
READS.fields.posted and falls back to the old set for undeclared readers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:31:45 +02:00
1 parent 4c0da58da7
commit bbd7522163
10 files changed
+1437 -98

No files matched your search

+313 -5
View File
@@ -19,8 +19,13 @@ several hundred milliseconds a query, as a set intersection on the words of a
line it costs a few. `register` gives a connection that answer as the SQL
function `place_match`, and `match_sql` returns the clause that calls it.
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
chip's text back from the address. This module opens no database; the caller
hands over the places it holds, or its connection.
chip's text back from the address.
The filter and the gates no longer match words: a role's places are resolved
against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
and a token is the places under the place it names. `matches` stays the rule
for a `place:` token the gazetteer does not know. This module opens no
database; the caller hands over the places it holds, or its connection.
"""
from __future__ import annotations
@@ -32,10 +37,12 @@ from collections.abc import Iterable
from dataclasses import dataclass
from functools import lru_cache
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
country_of)
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed",
"Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse",
"placeless", "register", "seed", "suggest", "towns"]
def fold(text: str) -> str:
@@ -538,6 +545,9 @@ _PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Danish boards append "and the possibility of working from home".
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
#: A slash between two places, "Malmö/Lund", "Berlin / Munich".
_SLASH = re.compile(r"\s*/\s*")
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
@@ -665,3 +675,301 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
regions.sort(key=lambda s: (-s.count, s.label))
towns.sort(key=lambda s: (-s.count, s.label))
return (countries + regions + towns)[:limit]
# ------------------------------------------------------------------ gazetteer
#: The kinds of place, widest first: the index is how specific a kind is.
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
#: Entries of `_CITIES` that name a region rather than a town.
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
"bay area"))
#: Words a place text holds when it names how or where-ever a role is worked
#: rather than a place: "Remote", "Multiple Locations", "Home based". A text
#: of these alone is no place, so it is never listed as unresolved.
_PLACELESS = frozenset((
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
def placeless(text: str) -> bool:
"""Whether a place text names no place at all: empty, or only numbers and
words such as `Remote`, `Hybrid`, `2 Locations`."""
return all(word in _PLACELESS or word.isdigit() for word in fold(text).split())
@dataclass(frozen=True)
class Seed:
"""One place as the code knows it: `key` is unique within `kind` (an ISO
code for a country, a region's key, a town's group), `parent` the
`(kind, key)` of the place it lies in, `aliases` the folds it is written
as."""
kind: str
key: str
name: str
parent: tuple[str, str] | None
aliases: tuple[str, ...]
def _subdivisions() -> list[tuple[str, str, str]]:
"""`(country, code, name)` of every US state and Canadian province."""
found = []
for line in _SUBDIVISIONS.split("\n"):
if line:
country, rest = line.split(" ", 1)
for entry in rest.split("|"):
name, code = entry.strip().rsplit(" ", 1)
found.append((country, code, name))
return found
def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
"""The gazetteer the dictionaries in code describe, parents before their
children, and the inclusions beyond the tree (Øresund holds the Capital
Region and Skåne, which lie in two countries).
Europe is a continent over the countries of `_EUROPE`, and is also what
`EU`, `EMEA` and `European Union` name. Every country of `_COUNTRY_NAMES`
by its names; the US states and Canadian provinces as regions; the
regions of `REGIONS`; the towns of the alias groups, of the regions and
of `_CITIES`, one place per group (`Copenhagen`, `København`,
`Kopenhamn` are one), in their region when a region lists them and else
in their country."""
europe = ("continent", "europe")
seeds: dict[tuple[str, str], Seed] = {}
def add(kind: str, key: str, name: str, parent: tuple[str, str] | None,
aliases: Iterable[str]) -> None:
folds = tuple(dict.fromkeys(a for a in aliases if a))
if (kind, key) in seeds:
old = seeds[(kind, key)]
folds = tuple(dict.fromkeys((*old.aliases, *folds)))
seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds)
else:
seeds[(kind, key)] = Seed(kind, key, name, parent, folds)
add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS)))
for code, names in _COUNTRY_LABELS.items():
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
for country, code, name in _subdivisions():
add("region", f"{country}-{code}", name, ("country", country), (fold(name),))
for region in REGIONS.values():
parent = ("country", region.country) if region.country else europe
add("region", region.key, region.label, parent,
(v for name in (region.key, region.label, *region.names)
for v in _variants(fold(name))))
for name in sorted(_CITY_REGIONS):
add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,))
# Towns, one per group, in the region that lists them first.
town_region: dict[str, str] = {}
for region in REGIONS.values():
for name in region.towns:
town_region.setdefault(_group(fold(name)), region.key)
folds: dict[str, dict[str, None]] = {}
shown: dict[str, str] = {}
def add_town(folded: str, name: str) -> None:
key = _group(folded)
folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded)))
shown.setdefault(key, name)
for spellings in _ALIAS_GROUPS:
for spelling in spellings:
add_town(fold(spelling), spellings[0])
for region in REGIONS.values():
for name in region.towns:
add_town(fold(name), name)
for city in _CITIES:
if city not in _CITY_REGIONS and city not in _REGION_BY_NAME:
add_town(city, _TOWN_LABELS.get(city) or city.title())
for key, forms in folds.items():
country = next((_CITIES[f] for f in forms if f in _CITIES), "")
if key in town_region:
parent = ("region", town_region[key])
elif country:
parent = ("country", country)
else:
raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES")
add("city", key, shown[key], parent, forms)
includes = [(("region", region.key), ("region", part))
for region in REGIONS.values() for part in region.includes]
return list(seeds.values()), includes
@dataclass(frozen=True)
class Place:
"""A row of `places`."""
id: int
kind: str
key: str
name: str
parent_id: int | None
class Gazetteer:
"""The places, their aliases and what lies in what, as the database holds
them, read into memory: what resolves a role's place text to places and a
filter token to the places under it. Built by the store from its tables,
and rebuilt when they change."""
def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]],
aliases: Iterable[tuple[str, int]],
includes: Iterable[tuple[int, int]] = ()) -> None:
self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows}
self.by_key: dict[tuple[str, str], int] = {
(p.kind, p.key): p.id for p in self.places.values()}
by_fold: dict[str, list[int]] = {}
for folded, place_id in aliases:
by_fold.setdefault(folded, []).append(place_id)
self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()}
self._longest = max((len(f.split()) for f in self.aliases), default=1)
self._up: dict[int, set[int]] = {}
self._down: dict[int, set[int]] = {}
for p in self.places.values():
if p.parent_id is not None:
self._link(p.parent_id, p.id)
for whole, part in includes:
self._link(whole, part)
self._ancestors: dict[int, frozenset[int]] = {}
self._resolved: dict[str, tuple[int, ...]] = {}
def _link(self, whole: int, part: int) -> None:
self._up.setdefault(part, set()).add(whole)
self._down.setdefault(whole, set()).add(part)
def ancestors(self, place_id: int) -> frozenset[int]:
"""Every place `place_id` lies in, through parents and inclusions."""
if (found := self._ancestors.get(place_id)) is None:
seen: set[int] = set()
todo = list(self._up.get(place_id, ()))
while todo:
up = todo.pop()
if up not in seen:
seen.add(up)
todo += self._up.get(up, ())
found = self._ancestors[place_id] = frozenset(seen)
return found
def under(self, ids: Iterable[int]) -> frozenset[int]:
"""The places and every place that lies in one of them."""
seen: set[int] = set()
todo = list(ids)
while todo:
place_id = todo.pop()
if place_id not in seen:
seen.add(place_id)
todo += self._down.get(place_id, ())
return frozenset(seen)
def country(self, place_id: int) -> str:
"""The ISO code of the country a place is in, '' above or outside one."""
place: Place | None = self.places.get(place_id)
while place is not None and place.kind != "country":
place = self.places.get(place.parent_id) if place.parent_id is not None else None
return place.key if place is not None else ""
def codes(self, ids: Iterable[int]) -> frozenset[str]:
"""The ISO codes of the countries among `ids`, and `EU` with Europe:
what a `roles.country` column holds for a role of one of them."""
found = set()
for place_id in ids:
place = self.places[place_id]
if place.kind == "country":
found.add(place.key)
elif (place.kind, place.key) == ("continent", "europe"):
found.add("EU")
return frozenset(found)
def _specific(self, place_id: int) -> int:
return KINDS.index(self.places[place_id].kind)
def _hits(self, folded: str) -> list[tuple[int, ...]]:
words = folded.split()
hits = []
i = 0
while i < len(words):
for n in range(min(self._longest, len(words) - i), 0, -1):
if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None:
hits.append(ids)
i += n
break
else:
i += 1
return hits
def resolve(self, text: str) -> tuple[int, ...]:
"""The places one place text names, most specific first from the left:
`Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund`
is Malmö and Lund, `Paderborn, Germany` is Germany, `Remote` is none.
Every alias the text holds as whole words is a candidate, longest
first. A fold that names several places is the one in the country
`country_of` reads off the text, if one is, and else the most specific
(`Luxembourg` the town before the country). A candidate that another
one lies in is dropped, and so is one outside the one country the text
names, or, naming none, outside the regions it names in one country:
`London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the
Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several
countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country
`country_of` reads (from a state code, `Paris, TX`, or a code in
parentheses, `Remote (DE)`), the text is that country."""
folded = fold(text)
if (found := self._resolved.get(folded)) is None:
parts = _SLASH.split(_AND_HOME.sub("", text or ""))
found = tuple(dict.fromkeys(
p for part in parts for sub in _PART.split(part) for p in self._resolve(sub)))
self._resolved[folded] = found
return found
def _resolve(self, text: str) -> list[int]:
"""`resolve` for one part of a text, cut where `or`, `/` or `;` end one
place."""
code = country_of(text)
country = self.by_key.get(("country", code)) if code and code != "EU" else None
within = self.under((country,)) if country is not None else frozenset()
picked: list[int] = []
for hit in self._hits(fold(text)):
choice = [p for p in hit if p in within] or list(hit)
best = max(choice, key=self._specific)
if best not in picked:
picked.append(best)
# The widest units named hold the rest: one country, else the regions
# of one country. Several countries are a list, not a path.
for kind in ("country", "region"):
named = [p for p in picked if self.places[p].kind == kind]
if len({self.country(p) for p in named}) == 1:
holds = self.under(named)
picked = [p for p in picked if p in holds or self.places[p].kind == "continent"
or any(p in self.ancestors(q) for q in named)]
if named:
break
wider = {a for p in picked for a in self.ancestors(p)}
picked = [p for p in picked if p not in wider]
if country is not None and not any(p in within or p in self.ancestors(country)
for p in picked):
picked = [country]
return picked
def lookup(self, token: str) -> frozenset[int] | None:
"""The places a filter token names, before what lies in them: `country:DK`
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
every place written so (a town, or a region or a country by name).
None when the gazetteer knows no such place."""
tok = parse(token)
if tok.kind == "country":
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
found = {self.by_key[key]} if key in self.by_key else set()
elif tok.kind == "region":
found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \
else set(self.aliases.get(tok.value, ()))
else:
found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())}
return frozenset(found) or None