Match places with a registered SQL function, suggest regions on their own names, fold lines that hold a town into it
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
ebb1f560de
commit
1898f2a8a6
2 files changed
+947
-86
No files matched your search
+181
-64
@@ -13,23 +13,29 @@ of three tokens, and several tokens are joined by OR:
|
||||
place:malmo the town's fold and its aliases, so `copenhagen`,
|
||||
`kobenhavn` and `kopenhamn` find each other
|
||||
|
||||
`match_sql` turns a token into SQL over `roles.place_fold` and `roles.country`;
|
||||
`matches` answers whether one role's folded place and country fit a token, in
|
||||
Python: a region is over a hundred phrases, and as `LIKE` clauses that costs
|
||||
several hundred milliseconds a query, as a set intersection on the words of a
|
||||
line it costs a few. `register` gives a connection that answer as the SQL
|
||||
function `place_match`, and `match_sql` returns the clause that calls it.
|
||||
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
|
||||
chip's text back from the address. No database access here, the caller hands
|
||||
over the places it holds.
|
||||
chip's text back from the address. This module opens no database; the caller
|
||||
hands over the places it holds, or its connection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
|
||||
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
|
||||
|
||||
__all__ = ["REGIONS", "Region", "Suggestion", "Token", "fold", "label", "match_sql",
|
||||
"parse", "suggest"]
|
||||
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
|
||||
"label", "match_sql", "matches", "parse", "register", "suggest"]
|
||||
|
||||
|
||||
def fold(text: str) -> str:
|
||||
@@ -202,14 +208,16 @@ def _expand(text: str) -> tuple[str, ...]:
|
||||
class Region:
|
||||
"""A region below a country, or across two. `names` are what a board calls
|
||||
the region itself, `towns` the places in it; a line is in the region when it
|
||||
has one of either as whole words. Spellings are shown with their marks and
|
||||
folded when used."""
|
||||
has one of either as whole words, or is in a region it `includes`, by key.
|
||||
Only `names` are what the region is suggested and read as. Spellings are
|
||||
shown with their marks and folded when used."""
|
||||
|
||||
key: str
|
||||
label: str
|
||||
names: tuple[str, ...]
|
||||
towns: tuple[str, ...]
|
||||
country: str = ""
|
||||
includes: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _split(text: str) -> tuple[str, ...]:
|
||||
@@ -281,12 +289,12 @@ _FUNEN = Region(
|
||||
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
|
||||
"DK")
|
||||
|
||||
# The Øresund region is the two capital regions either side of the sound.
|
||||
# The Øresund region is the two capital regions either side of the sound. It is
|
||||
# suggested on its own names only; a line in either region is in it.
|
||||
_ORESUND = Region(
|
||||
"oresund", "Øresund",
|
||||
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen")
|
||||
+ _HOVEDSTADEN.names + _SKANE.names,
|
||||
_HOVEDSTADEN.towns + _SKANE.towns)
|
||||
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"),
|
||||
(), includes=("hovedstaden", "skane"))
|
||||
|
||||
REGIONS: dict[str, Region] = {
|
||||
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)}
|
||||
@@ -294,11 +302,13 @@ REGIONS: dict[str, Region] = {
|
||||
|
||||
def _phrases(region: Region) -> tuple[str, ...]:
|
||||
"""Every folded phrase that puts a line in the region, with the other
|
||||
spellings of its towns."""
|
||||
spellings of its towns, and those of the regions it includes."""
|
||||
found: dict[str, None] = {}
|
||||
for text in (*region.names, *region.towns):
|
||||
for form in _expand(fold(text)):
|
||||
found[form] = None
|
||||
for key in region.includes:
|
||||
found.update(dict.fromkeys(_phrases(REGIONS[key])))
|
||||
return tuple(found)
|
||||
|
||||
|
||||
@@ -321,6 +331,24 @@ for _region in REGIONS.values():
|
||||
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
|
||||
|
||||
|
||||
def _group(folded: str) -> str:
|
||||
"""The key of the town a fold is: the first spelling of its alias group, else
|
||||
the fold with `aa` as `a`."""
|
||||
return _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
|
||||
|
||||
|
||||
def _known_towns() -> dict[str, str]:
|
||||
"""Every town the tables name, as a fold, to its group: the alias groups, the
|
||||
towns of the regions and the cities `country_of` knows."""
|
||||
names = [*_GROUPS, *(fold(city) for city in _CITIES),
|
||||
*(fold(town) for region in REGIONS.values() for town in region.towns)]
|
||||
return {form: _group(form) for name in names for form in _variants(name)}
|
||||
|
||||
|
||||
_KNOWN = _known_towns()
|
||||
_KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- tokens
|
||||
|
||||
|
||||
@@ -369,57 +397,118 @@ def label(token: str | Token) -> str:
|
||||
return (token if isinstance(token, Token) else parse(token)).label
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ SQL
|
||||
# ------------------------------------------------------------------- matching
|
||||
|
||||
_NEVER = ("0", [])
|
||||
SQL_FUNCTION = "place_match"
|
||||
|
||||
|
||||
def _escape(word: str) -> str:
|
||||
"""A folded word holds no wildcard, but a caller may hand over raw text."""
|
||||
return re.sub(r"([\\%_])", r"\\\1", word)
|
||||
def known(token: str) -> bool:
|
||||
"""Whether the token names something: a country of the table (or `EU`), a
|
||||
region of `REGIONS`, a place with a value. `region:atlantis`, `country:ZZ`
|
||||
and `place:` are not."""
|
||||
return _known(parse(token))
|
||||
|
||||
|
||||
def _words(column: str, phrases: Iterable[str]) -> tuple[str, list]:
|
||||
"""Any of the phrases as whole words of the column. A multi-word phrase is
|
||||
a phrase: `kgs lyngby` needs both words, side by side."""
|
||||
phrases = [p for p in dict.fromkeys(phrases) if p]
|
||||
if not phrases:
|
||||
return _NEVER
|
||||
clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'"
|
||||
return " OR ".join([clause] * len(phrases)), [f"% {_escape(p)} %" for p in phrases]
|
||||
def _known(tok: Token) -> bool:
|
||||
if tok.kind == "country":
|
||||
return tok.value in _COUNTRY_LABELS or tok.value == "EU"
|
||||
if tok.kind == "region":
|
||||
return tok.value in REGIONS
|
||||
return bool(tok.value)
|
||||
|
||||
|
||||
def _country(code: str, place: str, country: str) -> tuple[str, list]:
|
||||
"""The role's own country, or a line that names the country while `country`
|
||||
is empty or `EU`: a board listing several offices has no single country."""
|
||||
names, params = _words(place, _COUNTRY_MENTIONS.get(code, ()))
|
||||
if not params:
|
||||
return f"{country} = ?", [code]
|
||||
return f"{country} = ? OR ({country} IN ('', 'EU') AND ({names}))", [code, *params]
|
||||
@dataclass(frozen=True)
|
||||
class _Matcher:
|
||||
"""What a token asks of a role. `phrases` are whole-word phrases of the
|
||||
folded place; `countries` are `(code, mentions)`: the role's own country, or
|
||||
a line with one of the mentions while `country` is empty or `EU`, since a
|
||||
board listing several offices has no single country."""
|
||||
|
||||
phrases: frozenset[str]
|
||||
countries: tuple[tuple[str, frozenset[str]], ...]
|
||||
longest: int
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def _matcher(token: str) -> _Matcher | None:
|
||||
"""The token read once: `None` when it names nothing.
|
||||
|
||||
A `place:` value that is also a region or a country is matched as that too:
|
||||
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
|
||||
written as a bare word means what the owner meant by it."""
|
||||
tok = parse(token)
|
||||
if not _known(tok):
|
||||
return None
|
||||
phrases: set[str] = set()
|
||||
countries: list[tuple[str, frozenset[str]]] = []
|
||||
if tok.kind == "country":
|
||||
countries.append((tok.value, frozenset(_COUNTRY_MENTIONS.get(tok.value, ()))))
|
||||
elif tok.kind == "region":
|
||||
phrases.update(_PHRASES[tok.value])
|
||||
else:
|
||||
phrases.update(_expand(tok.value))
|
||||
phrases.update(_PHRASES.get(_REGION_BY_NAME.get(tok.value, ""), ()))
|
||||
if code := _COUNTRY_BY_NAME.get(tok.value):
|
||||
countries.append((code, frozenset(_COUNTRY_MENTIONS.get(code, ()))))
|
||||
phrases.discard("")
|
||||
longest = max((len(p.split()) for p in (*phrases, *(m for _, ms in countries for m in ms))),
|
||||
default=1)
|
||||
return _Matcher(frozenset(phrases), tuple(countries), longest)
|
||||
|
||||
|
||||
@lru_cache(maxsize=32768)
|
||||
def _runs(place_fold: str, longest: int) -> frozenset[str]:
|
||||
"""Every run of up to `longest` words of a folded line, each a whole-word
|
||||
phrase the line holds."""
|
||||
words = place_fold.split()
|
||||
return frozenset(" ".join(words[i:i + n])
|
||||
for n in range(1, longest + 1) for i in range(len(words) - n + 1))
|
||||
|
||||
|
||||
def matches(place_fold: str, country: str, token: str) -> bool:
|
||||
"""Whether a role whose folded place and country are these is found by the
|
||||
token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or
|
||||
a Danish city) while `country` is empty or `EU`. `region:skane` is any of
|
||||
its names or towns as whole words, `place:malmo` the town and its aliases.
|
||||
A token that names nothing (`known` is false) matches no role."""
|
||||
match = _matcher(token)
|
||||
if match is None:
|
||||
return False
|
||||
runs = None
|
||||
if match.phrases:
|
||||
runs = _runs(place_fold, match.longest)
|
||||
if not match.phrases.isdisjoint(runs):
|
||||
return True
|
||||
for code, mentions in match.countries:
|
||||
if country == code:
|
||||
return True
|
||||
if mentions and country in ("", "EU"):
|
||||
runs = _runs(place_fold, match.longest) if runs is None else runs
|
||||
if not mentions.isdisjoint(runs):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _place_match(place_fold: str | None, country: str | None, token: str) -> int:
|
||||
return int(matches(place_fold or "", country or "", token))
|
||||
|
||||
|
||||
def register(conn: sqlite3.Connection) -> None:
|
||||
"""Give a connection `place_match(place_fold, country, token)`, which is 1 when
|
||||
`matches` is true. `match_sql` returns a clause that calls it."""
|
||||
conn.create_function(SQL_FUNCTION, 3, _place_match, deterministic=True)
|
||||
|
||||
|
||||
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
|
||||
-> tuple[str, list]:
|
||||
"""SQL over the two columns for one token, parenthesised, with `?` parameters.
|
||||
Several tokens are joined with OR by the caller. Bare text is a place.
|
||||
|
||||
A `place:` value that is also a region or a country is matched as that too:
|
||||
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
|
||||
written as a bare word means what the owner meant by it. A token that names
|
||||
nothing known (`region:atlantis`, an empty value) matches no role."""
|
||||
"""SQL over the two columns for one token, with one `?` parameter. It calls
|
||||
`place_match`, so the connection needs `register`. Several tokens are joined
|
||||
with OR by the caller. Bare text is a place. A token that names nothing
|
||||
(`known` is false) is `0`, which matches no role."""
|
||||
tok = parse(token)
|
||||
if tok.kind == "country":
|
||||
sql, params = _country(tok.value, place, country)
|
||||
elif tok.kind == "region":
|
||||
sql, params = _words(place, _PHRASES.get(tok.value, ()))
|
||||
else:
|
||||
region = _REGION_BY_NAME.get(tok.value)
|
||||
sql, params = _words(place, [*_expand(tok.value), *_PHRASES.get(region, ())]) \
|
||||
if tok.value else _NEVER
|
||||
if tok.value in _COUNTRY_BY_NAME:
|
||||
more, extra = _country(_COUNTRY_BY_NAME[tok.value], place, country)
|
||||
sql, params = f"{sql} OR {more}", [*params, *extra]
|
||||
return f"({sql})", params
|
||||
if not _known(tok):
|
||||
return "0", []
|
||||
return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- suggestions
|
||||
@@ -442,9 +531,9 @@ def _prefixes(q: str, candidates: Iterable[str]) -> bool:
|
||||
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
|
||||
|
||||
|
||||
#: Where one town ends in a line: `;` between places, and `or` / `eller` / `og`
|
||||
#: between two towns in one ("København V eller Aalborg").
|
||||
_PART = re.compile(r"\s*;\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
|
||||
#: Where one town ends in a line: `;` and `|` between places, and `or` / `eller` /
|
||||
#: `og` between two towns in one ("København V eller Aalborg").
|
||||
_PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
|
||||
|
||||
#: Danish boards append "and the possibility of working from home".
|
||||
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
|
||||
@@ -471,16 +560,32 @@ def _towns(place: str) -> list[str]:
|
||||
return found
|
||||
|
||||
|
||||
def _towns_in(folded: str) -> list[str]:
|
||||
"""The known towns a fold holds as whole words, longest first from the left,
|
||||
as their groups: `munich berlin` is Munich and Berlin, `copenhagen office` is
|
||||
Copenhagen, `hoje taastrup` is one town and not also Taastrup."""
|
||||
words = folded.split()
|
||||
found: list[str] = []
|
||||
i = 0
|
||||
while i < len(words):
|
||||
for n in range(min(_KNOWN_LONGEST, len(words) - i), 0, -1):
|
||||
if (group := _KNOWN.get(" ".join(words[i:i + n]))) is not None:
|
||||
found.append(group)
|
||||
i += n
|
||||
break
|
||||
else:
|
||||
i += 1
|
||||
return list(dict.fromkeys(found))
|
||||
|
||||
|
||||
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
|
||||
"""Open roles per region, over folded lines. A line is cut into every run of
|
||||
words once, and each region asks whether it holds one of its phrases."""
|
||||
sets = {key: set(_PHRASES[key]) for key in keys}
|
||||
longest = max((len(p.split()) for key in sets for p in sets[key]), default=1)
|
||||
sets = {key: frozenset(_PHRASES[key]) for key in keys}
|
||||
longest = max((len(p.split()) for phrases in sets.values() for p in phrases), default=1)
|
||||
counts: Counter[str] = Counter()
|
||||
for text, roles in places:
|
||||
words = text.split()
|
||||
runs = {" ".join(words[i:i + n])
|
||||
for n in range(1, longest + 1) for i in range(len(words) - n + 1)}
|
||||
runs = _runs(text, longest)
|
||||
for key, phrases in sets.items():
|
||||
if not phrases.isdisjoint(runs):
|
||||
counts[key] += roles
|
||||
@@ -516,20 +621,32 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
|
||||
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
|
||||
for key in named if counts[key]]
|
||||
|
||||
# A line's town is a group. A line that is a town is that group's spelling;
|
||||
# one that only holds a known town ("Copenhagen Office", "Munich - Berlin")
|
||||
# counts under each such town and shows as none; anything else is its own.
|
||||
counts_by_town: Counter[str] = Counter()
|
||||
spellings: dict[str, Counter[str]] = {}
|
||||
written: dict[str, set[str]] = {}
|
||||
for place, _, roles in rows:
|
||||
for town in _towns(place):
|
||||
folded = fold(town)
|
||||
group = _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
|
||||
if folded not in _KNOWN and (inside := _towns_in(folded)):
|
||||
for group in inside:
|
||||
counts_by_town[group] += roles
|
||||
continue
|
||||
group = _group(folded)
|
||||
counts_by_town[group] += roles
|
||||
spellings.setdefault(group, Counter())[town] += roles
|
||||
written.setdefault(group, set()).add(folded)
|
||||
towns = []
|
||||
for group, seen in spellings.items():
|
||||
if not _prefixes(text, [form for folded in written[group] for form in _expand(folded)]):
|
||||
for group, total in counts_by_town.items():
|
||||
forms = {group, *written.get(group, ())}
|
||||
if not _prefixes(text, [form for folded in forms for form in _expand(folded)]):
|
||||
continue
|
||||
shown = min(seen, key=lambda s: (-seen[s], s))
|
||||
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", sum(seen.values())))
|
||||
seen = spellings.get(group)
|
||||
shown = min(seen, key=lambda s: (-seen[s], s)) if seen \
|
||||
else _TOWN_LABELS.get(group) or group.title()
|
||||
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total))
|
||||
|
||||
countries.sort(key=lambda s: (-s.count, s.label))
|
||||
regions.sort(key=lambda s: (-s.count, s.label))
|
||||
|
||||
Reference in new issue
Block a user