Match places with a registered SQL function, suggest regions on their own names, fold lines that hold a town into it

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
WolfandClaude committed 2026-09-29 22:04:21 +02:00
1 parent ebb1f560de
commit 1898f2a8a6
2 files changed
+947 -86

No files matched your search

+181 -64
View File
@@ -13,23 +13,29 @@ of three tokens, and several tokens are joined by OR:
place:malmo the town's fold and its aliases, so `copenhagen`,
`kobenhavn` and `kopenhamn` find each other
`match_sql` turns a token into SQL over `roles.place_fold` and `roles.country`;
`matches` answers whether one role's folded place and country fit a token, in
Python: a region is over a hundred phrases, and as `LIKE` clauses that costs
several hundred milliseconds a query, as a set intersection on the words of a
line it costs a few. `register` gives a connection that answer as the SQL
function `place_match`, and `match_sql` returns the clause that calls it.
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
chip's text back from the address. No database access here, the caller hands
over the places it holds.
chip's text back from the address. This module opens no database; the caller
hands over the places it holds, or its connection.
"""
from __future__ import annotations
import re
import sqlite3
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass
from functools import lru_cache
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
__all__ = ["REGIONS", "Region", "Suggestion", "Token", "fold", "label", "match_sql",
"parse", "suggest"]
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
"label", "match_sql", "matches", "parse", "register", "suggest"]
def fold(text: str) -> str:
@@ -202,14 +208,16 @@ def _expand(text: str) -> tuple[str, ...]:
class Region:
"""A region below a country, or across two. `names` are what a board calls
the region itself, `towns` the places in it; a line is in the region when it
has one of either as whole words. Spellings are shown with their marks and
folded when used."""
has one of either as whole words, or is in a region it `includes`, by key.
Only `names` are what the region is suggested and read as. Spellings are
shown with their marks and folded when used."""
key: str
label: str
names: tuple[str, ...]
towns: tuple[str, ...]
country: str = ""
includes: tuple[str, ...] = ()
def _split(text: str) -> tuple[str, ...]:
@@ -281,12 +289,12 @@ _FUNEN = Region(
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
"DK")
# The Øresund region is the two capital regions either side of the sound.
# The Øresund region is the two capital regions either side of the sound. It is
# suggested on its own names only; a line in either region is in it.
_ORESUND = Region(
"oresund", "Øresund",
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen")
+ _HOVEDSTADEN.names + _SKANE.names,
_HOVEDSTADEN.towns + _SKANE.towns)
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"),
(), includes=("hovedstaden", "skane"))
REGIONS: dict[str, Region] = {
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)}
@@ -294,11 +302,13 @@ REGIONS: dict[str, Region] = {
def _phrases(region: Region) -> tuple[str, ...]:
"""Every folded phrase that puts a line in the region, with the other
spellings of its towns."""
spellings of its towns, and those of the regions it includes."""
found: dict[str, None] = {}
for text in (*region.names, *region.towns):
for form in _expand(fold(text)):
found[form] = None
for key in region.includes:
found.update(dict.fromkeys(_phrases(REGIONS[key])))
return tuple(found)
@@ -321,6 +331,24 @@ for _region in REGIONS.values():
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
def _group(folded: str) -> str:
"""The key of the town a fold is: the first spelling of its alias group, else
the fold with `aa` as `a`."""
return _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
def _known_towns() -> dict[str, str]:
"""Every town the tables name, as a fold, to its group: the alias groups, the
towns of the regions and the cities `country_of` knows."""
names = [*_GROUPS, *(fold(city) for city in _CITIES),
*(fold(town) for region in REGIONS.values() for town in region.towns)]
return {form: _group(form) for name in names for form in _variants(name)}
_KNOWN = _known_towns()
_KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN)
# --------------------------------------------------------------------- tokens
@@ -369,57 +397,118 @@ def label(token: str | Token) -> str:
return (token if isinstance(token, Token) else parse(token)).label
# ------------------------------------------------------------------------ SQL
# ------------------------------------------------------------------- matching
_NEVER = ("0", [])
SQL_FUNCTION = "place_match"
def _escape(word: str) -> str:
"""A folded word holds no wildcard, but a caller may hand over raw text."""
return re.sub(r"([\\%_])", r"\\\1", word)
def known(token: str) -> bool:
"""Whether the token names something: a country of the table (or `EU`), a
region of `REGIONS`, a place with a value. `region:atlantis`, `country:ZZ`
and `place:` are not."""
return _known(parse(token))
def _words(column: str, phrases: Iterable[str]) -> tuple[str, list]:
"""Any of the phrases as whole words of the column. A multi-word phrase is
a phrase: `kgs lyngby` needs both words, side by side."""
phrases = [p for p in dict.fromkeys(phrases) if p]
if not phrases:
return _NEVER
clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'"
return " OR ".join([clause] * len(phrases)), [f"% {_escape(p)} %" for p in phrases]
def _known(tok: Token) -> bool:
if tok.kind == "country":
return tok.value in _COUNTRY_LABELS or tok.value == "EU"
if tok.kind == "region":
return tok.value in REGIONS
return bool(tok.value)
def _country(code: str, place: str, country: str) -> tuple[str, list]:
"""The role's own country, or a line that names the country while `country`
is empty or `EU`: a board listing several offices has no single country."""
names, params = _words(place, _COUNTRY_MENTIONS.get(code, ()))
if not params:
return f"{country} = ?", [code]
return f"{country} = ? OR ({country} IN ('', 'EU') AND ({names}))", [code, *params]
@dataclass(frozen=True)
class _Matcher:
"""What a token asks of a role. `phrases` are whole-word phrases of the
folded place; `countries` are `(code, mentions)`: the role's own country, or
a line with one of the mentions while `country` is empty or `EU`, since a
board listing several offices has no single country."""
phrases: frozenset[str]
countries: tuple[tuple[str, frozenset[str]], ...]
longest: int
@lru_cache(maxsize=1024)
def _matcher(token: str) -> _Matcher | None:
"""The token read once: `None` when it names nothing.
A `place:` value that is also a region or a country is matched as that too:
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
written as a bare word means what the owner meant by it."""
tok = parse(token)
if not _known(tok):
return None
phrases: set[str] = set()
countries: list[tuple[str, frozenset[str]]] = []
if tok.kind == "country":
countries.append((tok.value, frozenset(_COUNTRY_MENTIONS.get(tok.value, ()))))
elif tok.kind == "region":
phrases.update(_PHRASES[tok.value])
else:
phrases.update(_expand(tok.value))
phrases.update(_PHRASES.get(_REGION_BY_NAME.get(tok.value, ""), ()))
if code := _COUNTRY_BY_NAME.get(tok.value):
countries.append((code, frozenset(_COUNTRY_MENTIONS.get(code, ()))))
phrases.discard("")
longest = max((len(p.split()) for p in (*phrases, *(m for _, ms in countries for m in ms))),
default=1)
return _Matcher(frozenset(phrases), tuple(countries), longest)
@lru_cache(maxsize=32768)
def _runs(place_fold: str, longest: int) -> frozenset[str]:
"""Every run of up to `longest` words of a folded line, each a whole-word
phrase the line holds."""
words = place_fold.split()
return frozenset(" ".join(words[i:i + n])
for n in range(1, longest + 1) for i in range(len(words) - n + 1))
def matches(place_fold: str, country: str, token: str) -> bool:
"""Whether a role whose folded place and country are these is found by the
token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or
a Danish city) while `country` is empty or `EU`. `region:skane` is any of
its names or towns as whole words, `place:malmo` the town and its aliases.
A token that names nothing (`known` is false) matches no role."""
match = _matcher(token)
if match is None:
return False
runs = None
if match.phrases:
runs = _runs(place_fold, match.longest)
if not match.phrases.isdisjoint(runs):
return True
for code, mentions in match.countries:
if country == code:
return True
if mentions and country in ("", "EU"):
runs = _runs(place_fold, match.longest) if runs is None else runs
if not mentions.isdisjoint(runs):
return True
return False
def _place_match(place_fold: str | None, country: str | None, token: str) -> int:
return int(matches(place_fold or "", country or "", token))
def register(conn: sqlite3.Connection) -> None:
"""Give a connection `place_match(place_fold, country, token)`, which is 1 when
`matches` is true. `match_sql` returns a clause that calls it."""
conn.create_function(SQL_FUNCTION, 3, _place_match, deterministic=True)
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
-> tuple[str, list]:
"""SQL over the two columns for one token, parenthesised, with `?` parameters.
Several tokens are joined with OR by the caller. Bare text is a place.
A `place:` value that is also a region or a country is matched as that too:
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
written as a bare word means what the owner meant by it. A token that names
nothing known (`region:atlantis`, an empty value) matches no role."""
"""SQL over the two columns for one token, with one `?` parameter. It calls
`place_match`, so the connection needs `register`. Several tokens are joined
with OR by the caller. Bare text is a place. A token that names nothing
(`known` is false) is `0`, which matches no role."""
tok = parse(token)
if tok.kind == "country":
sql, params = _country(tok.value, place, country)
elif tok.kind == "region":
sql, params = _words(place, _PHRASES.get(tok.value, ()))
else:
region = _REGION_BY_NAME.get(tok.value)
sql, params = _words(place, [*_expand(tok.value), *_PHRASES.get(region, ())]) \
if tok.value else _NEVER
if tok.value in _COUNTRY_BY_NAME:
more, extra = _country(_COUNTRY_BY_NAME[tok.value], place, country)
sql, params = f"{sql} OR {more}", [*params, *extra]
return f"({sql})", params
if not _known(tok):
return "0", []
return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)]
# ---------------------------------------------------------------- suggestions
@@ -442,9 +531,9 @@ def _prefixes(q: str, candidates: Iterable[str]) -> bool:
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
#: Where one town ends in a line: `;` between places, and `or` / `eller` / `og`
#: between two towns in one ("København V eller Aalborg").
_PART = re.compile(r"\s*;\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Where one town ends in a line: `;` and `|` between places, and `or` / `eller` /
#: `og` between two towns in one ("København V eller Aalborg").
_PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Danish boards append "and the possibility of working from home".
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
@@ -471,16 +560,32 @@ def _towns(place: str) -> list[str]:
return found
def _towns_in(folded: str) -> list[str]:
"""The known towns a fold holds as whole words, longest first from the left,
as their groups: `munich berlin` is Munich and Berlin, `copenhagen office` is
Copenhagen, `hoje taastrup` is one town and not also Taastrup."""
words = folded.split()
found: list[str] = []
i = 0
while i < len(words):
for n in range(min(_KNOWN_LONGEST, len(words) - i), 0, -1):
if (group := _KNOWN.get(" ".join(words[i:i + n]))) is not None:
found.append(group)
i += n
break
else:
i += 1
return list(dict.fromkeys(found))
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
"""Open roles per region, over folded lines. A line is cut into every run of
words once, and each region asks whether it holds one of its phrases."""
sets = {key: set(_PHRASES[key]) for key in keys}
longest = max((len(p.split()) for key in sets for p in sets[key]), default=1)
sets = {key: frozenset(_PHRASES[key]) for key in keys}
longest = max((len(p.split()) for phrases in sets.values() for p in phrases), default=1)
counts: Counter[str] = Counter()
for text, roles in places:
words = text.split()
runs = {" ".join(words[i:i + n])
for n in range(1, longest + 1) for i in range(len(words) - n + 1)}
runs = _runs(text, longest)
for key, phrases in sets.items():
if not phrases.isdisjoint(runs):
counts[key] += roles
@@ -516,20 +621,32 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
for key in named if counts[key]]
# A line's town is a group. A line that is a town is that group's spelling;
# one that only holds a known town ("Copenhagen Office", "Munich - Berlin")
# counts under each such town and shows as none; anything else is its own.
counts_by_town: Counter[str] = Counter()
spellings: dict[str, Counter[str]] = {}
written: dict[str, set[str]] = {}
for place, _, roles in rows:
for town in _towns(place):
folded = fold(town)
group = _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
if folded not in _KNOWN and (inside := _towns_in(folded)):
for group in inside:
counts_by_town[group] += roles
continue
group = _group(folded)
counts_by_town[group] += roles
spellings.setdefault(group, Counter())[town] += roles
written.setdefault(group, set()).add(folded)
towns = []
for group, seen in spellings.items():
if not _prefixes(text, [form for folded in written[group] for form in _expand(folded)]):
for group, total in counts_by_town.items():
forms = {group, *written.get(group, ())}
if not _prefixes(text, [form for folded in forms for form in _expand(folded)]):
continue
shown = min(seen, key=lambda s: (-seen[s], s))
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", sum(seen.values())))
seen = spellings.get(group)
shown = min(seen, key=lambda s: (-seen[s], s)) if seen \
else _TOWN_LABELS.get(group) or group.title()
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total))
countries.sort(key=lambda s: (-s.count, s.label))
regions.sort(key=lambda s: (-s.count, s.label))