Add places: folded whole-word place matching, country, region and town tokens, suggestions

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
WolfandClaude committed 2026-09-29 21:58:32 +02:00
1 parent 64f367cc18
commit ebb1f560de
2 files changed
+990

No files matched your search

+537
View File
@@ -0,0 +1,537 @@
"""Places as a filter: what a board writes in `place`, matched by whole words.
A location line is free text: "København K", "Kgs. Lyngby", "Aarhus N, Region
Central Jutland, Denmark". A substring match finds `Lund` in `Billund` and
`Denmark` in nothing that only names a town. Here a place is folded (casefolded,
marks stripped, `ø` to `o`, `æ` to `ae`, `å` to `a`, punctuation to spaces) and
matched as whole words, so `Lund` is `lund` and no other word. A filter is one
of three tokens, and several tokens are joined by OR:
country:DK the role's country, or a line that names it while `country`
is empty or `EU` (a board that lists several offices)
region:skane any town or name of a region in `REGIONS`
place:malmo the town's fold and its aliases, so `copenhagen`,
`kobenhavn` and `kopenhamn` find each other
`match_sql` turns a token into SQL over `roles.place_fold` and `roles.country`;
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
chip's text back from the address. No database access here, the caller hands
over the places it holds.
"""
from __future__ import annotations
import re
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
__all__ = ["REGIONS", "Region", "Suggestion", "Token", "fold", "label", "match_sql",
"parse", "suggest"]
def fold(text: str) -> str:
"""`Skåne län` -> `skane lan`, `København K` -> `kobenhavn k`, `Kgs. Lyngby`
-> `kgs lyngby`. What `roles.place_fold` holds. `å` needs no rule of its own:
NFKD splits it into `a` and a ring, and the ring is a mark."""
return _place_fold(text)
# ------------------------------------------------------------------ countries
def _countries() -> dict[str, tuple[str, ...]]:
"""Every code to the names `_COUNTRY_NAMES` gives it, English name first."""
found: dict[str, tuple[str, ...]] = {}
for line in _COUNTRY_NAMES.split("\n"):
if line:
code, rest = line.split(" ", 1)
found[code] = tuple(name.strip() for name in rest.split("|"))
return found
_COUNTRY_LABELS = _countries()
#: A code to the folded names a country goes by.
_COUNTRY_FOLDS: dict[str, tuple[str, ...]] = {
code: tuple(dict.fromkeys(filter(None, map(fold, names))))
for code, names in _COUNTRY_LABELS.items()}
#: What a line has to say for a role of that country whose `country` column is
#: empty or `EU`: a name of the country, or a city `country_of` knows in it. A
#: board that lists offices ("Copenhagen; London; Warsaw") names cities only.
_COUNTRY_MENTIONS: dict[str, tuple[str, ...]] = {
code: tuple(dict.fromkeys((*names, *(fold(c) for c, cc in _CITIES.items() if cc == code))))
for code, names in _COUNTRY_FOLDS.items()}
#: A folded country name to its code.
_COUNTRY_BY_NAME: dict[str, str] = {
name: code for code, names in _COUNTRY_FOLDS.items() for name in names}
#: Words that are not a town: a country, a macro region, how the role is worked.
_NOT_A_TOWN = frozenset(_COUNTRY_BY_NAME) | frozenset(map(fold, _REGIONS)) \
| {"remote", "hybrid"}
# -------------------------------------------------------------------- aliases
#: Towns boards write in more than one language, first spelling shown. Each
#: group is symmetric: any spelling finds the others. The Nordics and the DACH
#: countries in full for their large cities, and the European cities that
#: appear both ways in boards.
_ALIAS_GROUPS: tuple[tuple[str, ...], ...] = (
# Denmark
("Copenhagen", "København", "Kobenhavn", "Köpenhamn", "Kopenhamn", "Kopenhagen",
"Copenhague", "Kopenhaga"),
("Aarhus", "Århus"),
("Aalborg", "Ålborg"),
("Kgs. Lyngby", "Kongens Lyngby", "Lyngby"),
("Taastrup", "Tåstrup"),
("Elsinore", "Helsingør"),
# Sweden
("Gothenburg", "Göteborg", "Goteborg"),
("Stockholm",),
("Malmö", "Malmo"),
("Helsingborg", "Hälsingborg"),
("Västerås", "Vasteras"),
("Linköping", "Linkoping"),
("Örebro", "Orebro"),
("Norrköping", "Norrkoping"),
("Jönköping", "Jonkoping"),
# Norway, Finland, Iceland
("Oslo",),
("Bergen",),
("Trondheim", "Trondhjem"),
("Helsinki", "Helsingfors"),
("Espoo", "Esbo"),
("Vantaa", "Vanda"),
("Reykjavik", "Reykjavík"),
# Germany
("Munich", "München", "Muenchen", "Munchen"),
("Cologne", "Köln", "Koeln", "Koln"),
("Nuremberg", "Nürnberg", "Nuernberg", "Nurnberg"),
("Düsseldorf", "Duesseldorf", "Dusseldorf"),
("Hanover", "Hannover"),
("Mainz", "Mayence"),
("Frankfurt", "Frankfurt am Main"),
("Göttingen", "Goettingen", "Gottingen"),
("Würzburg", "Wuerzburg", "Wurzburg"),
("Tübingen", "Tuebingen", "Tubingen"),
("Lübeck", "Luebeck", "Lubeck"),
("Münster", "Muenster", "Munster"),
# Austria
("Vienna", "Wien", "Vienne"),
("Salzburg",),
("Innsbruck",),
# Switzerland
("Zurich", "Zürich", "Zuerich"),
("Geneva", "Genève", "Geneve", "Genf", "Ginevra"),
("Basel", "Basle", "Bâle"),
("Bern", "Berne"),
("Lucerne", "Luzern"),
("St. Gallen", "Sankt Gallen", "Saint Gall"),
# The rest of Europe
("Prague", "Praha", "Prag"),
("Warsaw", "Warszawa", "Warschau"),
("Krakow", "Kraków", "Cracow"),
("Wroclaw", "Wrocław", "Breslau"),
("Gdansk", "Gdańsk", "Danzig"),
("Poznan", "Poznań", "Posen"),
("Lisbon", "Lisboa", "Lissabon"),
("Milan", "Milano", "Mailand"),
("Rome", "Roma", "Rom"),
("Turin", "Torino"),
("Florence", "Firenze"),
("Naples", "Napoli", "Neapel"),
("Venice", "Venezia", "Venedig"),
("Genoa", "Genova"),
("Brussels", "Bruxelles", "Brussel", "Brüssel"),
("Antwerp", "Antwerpen", "Anvers"),
("Ghent", "Gent", "Gand"),
("Bruges", "Brugge"),
("Liège", "Liege", "Luik", "Lüttich"),
("The Hague", "Den Haag", "s-Gravenhage"),
("Luxembourg", "Luxemburg"),
("Seville", "Sevilla"),
("Athens", "Athina", "Athen"),
("Bucharest", "București", "Bucuresti"),
("Belgrade", "Beograd"),
("Bratislava", "Pressburg"),
("Kyiv", "Kiev"),
("Cluj-Napoca", "Cluj"),
("Timișoara", "Timisoara"),
("Marseille", "Marseilles"),
("Bolzano", "Bozen"),
)
def _variants(text: str) -> tuple[str, ...]:
"""A fold and its `aa` spelling: Danish writes `å` as `aa` (`Aarhus`, `Taastrup`)
and `Århus` and `Tåstrup` fold to `arhus` and `tastrup`."""
return tuple(dict.fromkeys((text, text.replace("aa", "a"))))
def _aliases() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]:
"""Every folded spelling to its whole group (all folded), and to its shown
spelling. The group starts with the first spelling listed."""
groups: dict[str, tuple[str, ...]] = {}
shown: dict[str, str] = {}
for group in _ALIAS_GROUPS:
members = tuple(dict.fromkeys(v for name in group for v in _variants(fold(name))))
for name in group:
shown.setdefault(fold(name), name)
for member in members:
groups[member] = members
return groups, shown
_GROUPS, _SHOWN = _aliases()
def _expand(text: str) -> tuple[str, ...]:
"""A folded town and everything it is also written as."""
return _GROUPS.get(text) or _variants(text)
# -------------------------------------------------------------------- regions
@dataclass(frozen=True)
class Region:
"""A region below a country, or across two. `names` are what a board calls
the region itself, `towns` the places in it; a line is in the region when it
has one of either as whole words. Spellings are shown with their marks and
folded when used."""
key: str
label: str
names: tuple[str, ...]
towns: tuple[str, ...]
country: str = ""
def _split(text: str) -> tuple[str, ...]:
return tuple(part.strip() for part in text.split(",") if part.strip())
_SKANE = Region(
"skane", "Skåne",
_split("Skåne, Skåne län, Skåne County, Region Skåne, Skåne Region, Scania, Skaane"),
_split("""Malmö, Lund, Helsingborg, Kristianstad, Landskrona, Trelleborg, Ängelholm,
Hässleholm, Ystad, Eslöv, Höganäs, Staffanstorp, Lomma, Bjuv, Burlöv, Kävlinge,
Svedala, Vellinge, Simrishamn, Tomelilla, Sjöbo, Svalöv, Klippan, Åstorp,
Bromölla, Osby, Perstorp, Örkelljunga, Östra Göinge, Hörby, Höör, Båstad,
Skurup, Arlöv, Limhamn, Dalby, Åkarp, Bjärred, Höllviken, Ljungbyhed,
Löddeköpinge, Furulund, Genarp, Veberöd, Skanör, Falsterbo, Mölle, Arild"""),
"SE")
_HOVEDSTADEN = Region(
"hovedstaden", "Capital Region of Denmark",
_split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"),
_split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup,
Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte,
Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre,
Høje-Taastrup, Hørsholm, Ishøj, Lyngby-Taarbæk, Rudersdal, Rødovre, Tårnby,
Vallensbæk, Kgs. Lyngby, Kongens Lyngby, Lyngby, Virum, Holte, Nærum, Birkerød,
Vedbæk, Skodsborg, Klampenborg, Charlottenlund, Hellerup, Ordrup, Bagsværd,
Søborg, Mørkhøj, Måløv, Skovlunde, Smørum, Ølstykke, Stenløse, Værløse, Farum,
Slangerup, Hedehusene, Taastrup, Kastrup, Amager, Valby, Vanløse, Brønshøj,
Nørrebro, Østerbro, Vesterbro, Nordhavn, Ørestad, Sydhavn, Islands Brygge,
Refshaleøen, Humlebæk, Kokkedal, Espergærde, Snekkersten, Rungsted, Rungsted Kyst,
Lillerød, Frederiksværk, Hundested, Helsinge, Gilleleje, Græsted, Vallensbæk Strand,
Skævinge, Rønne, Nexø, Allinge, Svaneke, Aakirkeby, Hasle"""),
"DK")
# Region Sjælland, not the island: Copenhagen is on Zealand too, but a board that
# writes "Region Zealand" means the region and the Capital Region has its own key.
# The bare word is left out of `names` since "New Zealand" would match it.
_ZEALAND = Region(
"zealand", "Zealand",
_split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"),
_split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved,
Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg,
Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing,
Haslev, Korsør, Skælskør, Præstø, Stege, Fakse, Rødby, Rødbyhavn, Jyllinge,
Viby Sjælland, Solrød Strand, Fuglebjerg, Gørlev, Dianalund, Jyderup, Hvalsø,
Tølløse, Hørve, Fårevejle, Bjæverskov, Nørre Asmindrup"""),
"DK")
_JUTLAND = Region(
"jutland", "Jutland",
_split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland,
North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland,
East Jutland, West Jutland, Vestjylland, Østjylland"""),
_split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning,
Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov,
Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå,
Hjørring, Frederikshavn, Skagen, Thisted, Struer, Ikast, Brande, Varde, Ribe,
Tønder, Vojens, Hadsten, Hobro, Skanderborg, Odder, Hedensted, Vejen, Ebeltoft,
Lemvig, Rønde, Egå, Lystrup, Hinnerup, Tilst, Hammel, Galten, Sabro, Harlev,
Løsning, Højbjerg, Juelsminde, Nørresundby, Brønderslev, Aars, Nibe, Sæby, Løgstør, Tarm,
Skjern, Padborg, Rødekro, Toftlund, Bramming, Kruså, Brabrand, Ry, Gjern"""),
"DK")
_FUNEN = Region(
"funen", "Funen",
_split("Funen, Fyn, Region Fyn"),
_split("""Odense, Svendborg, Nyborg, Faaborg, Fåborg, Middelfart, Assens, Kerteminde,
Bogense, Langeskov, Ringe, Otterup, Munkebo, Vissenbjerg, Årslev, Tommerup,
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
"DK")
# The Øresund region is the two capital regions either side of the sound.
_ORESUND = Region(
"oresund", "Øresund",
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen")
+ _HOVEDSTADEN.names + _SKANE.names,
_HOVEDSTADEN.towns + _SKANE.towns)
REGIONS: dict[str, Region] = {
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)}
def _phrases(region: Region) -> tuple[str, ...]:
"""Every folded phrase that puts a line in the region, with the other
spellings of its towns."""
found: dict[str, None] = {}
for text in (*region.names, *region.towns):
for form in _expand(fold(text)):
found[form] = None
return tuple(found)
_PHRASES: dict[str, tuple[str, ...]] = {key: _phrases(r) for key, r in REGIONS.items()}
#: A folded key, label or name to the region key; a name in two regions ("Capital
#: Region of Denmark" is also in Øresund) belongs to the one that lists it first.
_REGION_BY_NAME: dict[str, str] = {}
for _key, _region in REGIONS.items():
for _name in (_region.key, _region.label, *_region.names):
_REGION_BY_NAME.setdefault(fold(_name), _key)
#: What a place is shown as when a chip is read from the address: every alias
#: spelling, every region and every town of one, as written above.
_TOWN_LABELS: dict[str, str] = dict(_SHOWN)
for _region in REGIONS.values():
for _town in (_region.label, *_region.towns, *_region.names):
_TOWN_LABELS.setdefault(fold(_town), _town)
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
# --------------------------------------------------------------------- tokens
@dataclass(frozen=True)
class Token:
"""One place filter. `value` is a country code, a region key or a fold."""
kind: str
value: str
label: str
def __str__(self) -> str:
return f"{self.kind}:{self.value}"
def _region_key(text: str) -> str:
folded = fold(text)
return _REGION_BY_NAME.get(folded, folded)
def parse(token: str) -> Token:
"""`country:DK`, `region:skane` or `place:malmo`. Bare text, or a prefix that
is none of these, is a place: `Skåne` is `place:skane`. A country by name
(`country:Denmark`) and a region by label (`region:Skåne`) are read as well."""
kind, sep, value = (token or "").partition(":")
kind = kind.strip().lower()
if not sep or kind not in ("country", "region", "place"):
kind, value = "place", token or ""
value = value.strip()
if kind == "country":
code = value.upper()
if code not in _COUNTRY_LABELS:
code = _COUNTRY_BY_NAME.get(fold(value), code)
return Token("country", code, _COUNTRY_LABELS.get(code, (code,))[0])
if kind == "region":
key = _region_key(value)
return Token("region", key, REGIONS[key].label if key in REGIONS else key)
folded = fold(value)
return Token("place", folded, _TOWN_LABELS.get(folded) or folded.title())
def label(token: str | Token) -> str:
"""The text of a chip read from the address: `country:DK` is `Denmark`,
`region:skane` is `Skåne`, `place:malmo` is `Malmö`. A town the tables do not
know is its fold in title case."""
return (token if isinstance(token, Token) else parse(token)).label
# ------------------------------------------------------------------------ SQL
_NEVER = ("0", [])
def _escape(word: str) -> str:
"""A folded word holds no wildcard, but a caller may hand over raw text."""
return re.sub(r"([\\%_])", r"\\\1", word)
def _words(column: str, phrases: Iterable[str]) -> tuple[str, list]:
"""Any of the phrases as whole words of the column. A multi-word phrase is
a phrase: `kgs lyngby` needs both words, side by side."""
phrases = [p for p in dict.fromkeys(phrases) if p]
if not phrases:
return _NEVER
clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'"
return " OR ".join([clause] * len(phrases)), [f"% {_escape(p)} %" for p in phrases]
def _country(code: str, place: str, country: str) -> tuple[str, list]:
"""The role's own country, or a line that names the country while `country`
is empty or `EU`: a board listing several offices has no single country."""
names, params = _words(place, _COUNTRY_MENTIONS.get(code, ()))
if not params:
return f"{country} = ?", [code]
return f"{country} = ? OR ({country} IN ('', 'EU') AND ({names}))", [code, *params]
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
-> tuple[str, list]:
"""SQL over the two columns for one token, parenthesised, with `?` parameters.
Several tokens are joined with OR by the caller. Bare text is a place.
A `place:` value that is also a region or a country is matched as that too:
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
written as a bare word means what the owner meant by it. A token that names
nothing known (`region:atlantis`, an empty value) matches no role."""
tok = parse(token)
if tok.kind == "country":
sql, params = _country(tok.value, place, country)
elif tok.kind == "region":
sql, params = _words(place, _PHRASES.get(tok.value, ()))
else:
region = _REGION_BY_NAME.get(tok.value)
sql, params = _words(place, [*_expand(tok.value), *_PHRASES.get(region, ())]) \
if tok.value else _NEVER
if tok.value in _COUNTRY_BY_NAME:
more, extra = _country(_COUNTRY_BY_NAME[tok.value], place, country)
sql, params = f"{sql} OR {more}", [*params, *extra]
return f"({sql})", params
# ---------------------------------------------------------------- suggestions
@dataclass(frozen=True)
class Suggestion:
"""`token` is what the chip holds, `kind` is `country`, `region` or `place`,
`count` the open roles it stands for."""
token: str
label: str
kind: str
count: int
def _prefixes(q: str, candidates: Iterable[str]) -> bool:
"""`q` starts a word of a candidate, or a run of words: `kgs ly` finds
`kgs lyngby`, `hague` finds `the hague`."""
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
#: Where one town ends in a line: `;` between places, and `or` / `eller` / `og`
#: between two towns in one ("København V eller Aalborg").
_PART = re.compile(r"\s*;\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Danish boards append "and the possibility of working from home".
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
#: A country or "Remote" ahead of the town, "Sweden - Lund", "Remote-New Zealand".
_LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$")
def _towns(place: str) -> list[str]:
"""The first comma part of each place of a line: `Aarhus N, Region Central
Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal
district is dropped, since `place:kobenhavn` finds `København K` anyway."""
found = []
for part in _PART.split(_AND_HOME.sub("", place or "")):
head = part.split(",")[0].strip()
if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN:
head = lead[2]
head = _DISTRICT.sub("", head)
if fold(head) and fold(head) not in _NOT_A_TOWN:
found.append(head)
return found
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
"""Open roles per region, over folded lines. A line is cut into every run of
words once, and each region asks whether it holds one of its phrases."""
sets = {key: set(_PHRASES[key]) for key in keys}
longest = max((len(p.split()) for key in sets for p in sets[key]), default=1)
counts: Counter[str] = Counter()
for text, roles in places:
words = text.split()
runs = {" ".join(words[i:i + n])
for n in range(1, longest + 1) for i in range(len(words) - n + 1)}
for key, phrases in sets.items():
if not phrases.isdisjoint(runs):
counts[key] += roles
return counts
def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
-> list[Suggestion]:
"""Tokens for the text typed so far. `places` is `(place, country, open
roles)`. Countries first, then regions, then towns, each by open roles.
A country is suggested when `q` starts a word of one of its names, with the
roles that carry its code. A region likewise on its label and names, with the
roles whose line is in it. A town is the first comma part of each `;` part of
a line, grouped by fold and by alias (`Copenhagen` and `København` are one
town, shown as its commonest spelling). Countries, regions and towns with no
open role are left out: a chip that finds nothing helps nobody."""
text = fold(q)
if not text:
return []
rows = [(place or "", country or "", int(roles)) for place, country, roles in places]
by_country: Counter[str] = Counter()
for _, country, roles in rows:
by_country[country] += roles
countries = [Suggestion(f"country:{code}", names[0], "country", by_country[code])
for code, names in _COUNTRY_LABELS.items()
if by_country[code] and _prefixes(text, _COUNTRY_FOLDS[code])]
named = [key for key, r in REGIONS.items()
if _prefixes(text, [fold(r.label), *map(fold, r.names)])]
counts = _region_counts(named, [(fold(place), roles) for place, _, roles in rows])
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
for key in named if counts[key]]
spellings: dict[str, Counter[str]] = {}
written: dict[str, set[str]] = {}
for place, _, roles in rows:
for town in _towns(place):
folded = fold(town)
group = _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
spellings.setdefault(group, Counter())[town] += roles
written.setdefault(group, set()).add(folded)
towns = []
for group, seen in spellings.items():
if not _prefixes(text, [form for folded in written[group] for form in _expand(folded)]):
continue
shown = min(seen, key=lambda s: (-seen[s], s))
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", sum(seen.values())))
countries.sort(key=lambda s: (-s.count, s.label))
regions.sort(key=lambda s: (-s.count, s.label))
towns.sort(key=lambda s: (-s.count, s.label))
return (countries + regions + towns)[:limit]
+453
View File
@@ -0,0 +1,453 @@
"""`app.places`: folding, the three token kinds run as SQL, and the suggestions.
The rows are `place` strings as boards write them, taken from the live database.
Each token's SQL runs against an in-memory table shaped like `roles`."""
from __future__ import annotations
import sqlite3
import pytest
from app import places
from app.places import REGIONS, Suggestion, Token, fold, label, match_sql, parse, suggest
#: (place, country) as a board wrote them and `country_of` read them.
ROWS = [
("København", "DK"), ("Copenhagen, Denmark", "DK"), ("Humlebæk, Denmark", "DK"),
("København K", "DK"), ("København Ø", "DK"), ("Ballerup", "DK"), ("Denmark", "DK"),
("Aarhus N, Region Central Jutland, Denmark", "DK"), ("Aarhus C, Denmark", "DK"),
("Billund, Denmark", "DK"), ("Kgs. Lyngby", "DK"), ("Kongens Lyngby", "DK"),
("Måløv, Capital Region of Denmark, Denmark", "DK"), ("Søborg", "DK"),
("Kalundborg, Region Zealand, Denmark", "DK"), ("Albertslund", "DK"),
("Odense, REGION_DNK 1852 South Denmark, Denmark", "DK"), ("Nakskov", "DK"),
("Nordhavn og mulighed for hjemmearbejde", "DK"), ("Remote OR Copenhagen", "DK"),
("Malmö", "SE"), ("Lund", "SE"), ("Sweden - Lund", "SE"), ("Helsingborg", "SE"),
("Malmö, Skåne County, Sweden", "SE"), ("Skåne län, Sweden", "SE"),
("Höganäs, Sweden", "SE"), ("Stockholm, Sweden", "SE"), ("Gothenburg, Sweden", "SE"),
("Aarhus, Denmark; København, Denmark; Stockholm, Sweden", "EU"),
("Copenhagen; London; Warsaw; Milano", "EU"),
("Copenhagen; Warsaw; New York", ""),
("Berlin, Germany; Munich, Germany", "DE"), ("Berlin, Germany", "DE"),
("München", "DE"), ("Munich, Bavaria, Germany", "DE"), ("Köln", "DE"), ("Cologne", "DE"),
("Zürich", "CH"), ("Zurich, Switzerland", "CH"), ("Genève", "CH"), ("Geneva", "CH"),
("Wien", "AT"), ("Vienna, Austria", "AT"),
("Auckland, New Zealand", "NZ"), ("Remote", ""), ("Hybrid", ""), ("Europe", ""),
("Remote - Denmark", "DK"), ("", ""),
]
@pytest.fixture()
def table() -> sqlite3.Connection:
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
db.executemany("INSERT INTO roles VALUES (?, ?, ?)",
[(place, fold(place), country) for place, country in ROWS])
return db
def found(db: sqlite3.Connection, *tokens: str) -> set[str]:
"""The places the tokens find, joined with OR as a caller would."""
parts = [match_sql(token) for token in tokens]
sql = " OR ".join(sql for sql, _ in parts)
params = [p for _, params in parts for p in params]
return {row[0] for row in db.execute(f"SELECT place FROM roles WHERE {sql}", params)}
# ------------------------------------------------------------------- folding
@pytest.mark.parametrize("text, folded", [
("Skåne län", "skane lan"),
("København K", "kobenhavn k"),
("Kgs. Lyngby", "kgs lyngby"),
("Malmö", "malmo"),
("Åbyhøj", "abyhoj"),
("Ærø", "aero"),
("Düsseldorf", "dusseldorf"),
("Zürich", "zurich"),
("Wrocław", "wroclaw"),
("Höje-Taastrup, Denmark", "hoje taastrup denmark"),
(" Aarhus N ", "aarhus n"),
("", ""),
])
def test_fold(text: str, folded: str) -> None:
assert fold(text) == folded
def test_fold_turns_a_ring_and_umlauts_into_the_plain_letter() -> None:
assert fold("å") == "a"
assert (fold("ä"), fold("ö"), fold("ü")) == ("a", "o", "u")
# ------------------------------------------------------------------- parsing
def test_parse_reads_the_three_kinds() -> None:
assert parse("country:DK") == Token("country", "DK", "Denmark")
assert parse("region:skane") == Token("region", "skane", "Skåne")
assert parse("place:malmo") == Token("place", "malmo", "Malmö")
def test_parse_reads_bare_text_as_a_place() -> None:
assert parse("Skåne") == Token("place", "skane", "Skåne")
assert parse("Kgs. Lyngby").value == "kgs lyngby"
assert parse("Foo: Bar").kind == "place"
def test_parse_is_forgiving_about_case_and_names() -> None:
assert parse("Country:dk").value == "DK"
assert parse("country:Denmark").value == "DK"
assert parse("region:Skåne").value == "skane"
assert parse("region:Capital Region of Denmark").value == "hovedstaden"
assert parse("place:København").value == "kobenhavn"
def test_a_token_round_trips_through_its_string() -> None:
for text in ("country:DK", "region:oresund", "place:kgs lyngby"):
assert str(parse(text)) == text
def test_label_reads_a_chip_from_the_address() -> None:
assert label("country:SE") == "Sweden"
assert label("region:hovedstaden") == "Capital Region of Denmark"
assert label("place:kobenhavn") == "København"
assert label("place:copenhagen") == "Copenhagen"
assert label("place:malmo") == "Malmö"
assert label("place:tromso") == "Tromso"
assert label(parse("place:kgs lyngby")) == "Kgs. Lyngby"
assert label("country:ZZ") == "ZZ"
assert label("region:atlantis") == "atlantis"
# ---------------------------------------------------------------------- SQL
def test_a_place_is_matched_on_whole_words(table: sqlite3.Connection) -> None:
lund = found(table, "place:lund")
assert lund == {"Lund", "Sweden - Lund"}
assert "Billund, Denmark" not in lund and "Albertslund" not in lund
assert found(table, "place:billund") == {"Billund, Denmark"}
def test_a_multi_word_place_is_a_phrase(table: sqlite3.Connection) -> None:
assert found(table, "place:kgs lyngby") == {"Kgs. Lyngby", "Kongens Lyngby"}
assert found(table, "place:capital denmark") == set()
assert found(table, "place:denmark capital") == set()
def test_diacritics_and_spellings_meet(table: sqlite3.Connection) -> None:
assert found(table, "place:Skåne") == found(table, "place:skane")
assert "Malmö" in found(table, "place:malmo") and "Malmö" in found(table, "place:Malmö")
assert {"København", "København K"} <= found(table, "place:kobenhavn")
COPENHAGENS = {"København", "Copenhagen, Denmark", "København K", "Remote OR Copenhagen"}
@pytest.mark.parametrize("spelling", ["copenhagen", "kobenhavn", "kopenhamn", "kopenhagen",
"København", "Köpenhamn"])
def test_copenhagen_is_every_spelling(table: sqlite3.Connection, spelling: str) -> None:
assert found(table, f"place:{spelling}") >= COPENHAGENS
@pytest.mark.parametrize("group", places._ALIAS_GROUPS, ids=lambda g: g[0])
def test_every_alias_finds_every_other(group: tuple[str, ...]) -> None:
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
db.executemany("INSERT INTO roles VALUES (?, ?, '')",
[(name, fold(name)) for name in group])
for spelling in group:
sql, params = match_sql(f"place:{spelling}")
hit = {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)}
assert hit == set(group), spelling
def test_no_spelling_belongs_to_two_groups() -> None:
seen: dict[str, str] = {}
for group in places._ALIAS_GROUPS:
for name in group:
assert seen.setdefault(fold(name), group[0]) == group[0], name
@pytest.mark.parametrize("pair", [("Munich", "München"), ("Cologne", "Köln"),
("Vienna", "Wien"), ("Zurich", "Zürich"),
("Geneva", "Genève")])
def test_a_city_written_two_ways_is_found_either_way(
table: sqlite3.Connection, pair: tuple[str, str]) -> None:
one, other = pair
hit = found(table, f"place:{one}")
assert hit == found(table, f"place:{other}")
words = {w for place in hit for w in fold(place).split()}
assert {fold(one), fold(other)} <= words
def test_aa_and_the_ring_are_one_letter(table: sqlite3.Connection) -> None:
hit = found(table, "place:århus")
assert hit == found(table, "place:aarhus") == {
"Aarhus N, Region Central Jutland, Denmark", "Aarhus C, Denmark",
"Aarhus, Denmark; København, Denmark; Stockholm, Sweden"}
def test_a_line_with_several_places_is_found_by_each(table: sqlite3.Connection) -> None:
line = "Aarhus, Denmark; København, Denmark; Stockholm, Sweden"
for tok in ("place:aarhus", "place:kobenhavn", "place:stockholm", "place:copenhagen"):
assert line in found(table, tok), tok
def test_several_tokens_are_joined_by_or(table: sqlite3.Connection) -> None:
hit = found(table, "country:DK", "region:skane")
assert {"København", "Malmö", "Lund"} <= hit
assert "Stockholm, Sweden" not in hit
def test_a_country_is_its_code(table: sqlite3.Connection) -> None:
hit = found(table, "country:DK")
assert {"København", "Denmark", "Ballerup", "Nakskov", "Odense, REGION_DNK 1852 South Denmark, Denmark"} <= hit
assert "Malmö" not in hit and "Berlin, Germany" not in hit
assert "Auckland, New Zealand" not in hit
def test_a_country_is_also_a_line_that_names_it_when_country_is_unset(
table: sqlite3.Connection) -> None:
hit = found(table, "country:DK")
# `country` is EU: a board listing offices in several countries.
assert "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" in hit
# Only cities are named, and one of them is Danish.
assert "Copenhagen; London; Warsaw; Milano" in hit
assert "Copenhagen; Warsaw; New York" in hit
# The same words under a country that is set are that country's business.
assert found(table, "country:SE") >= {"Stockholm, Sweden", "Malmö"}
assert "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" in found(table, "country:SE")
assert "Copenhagen, Denmark" in hit # its own country is DK
def test_a_country_name_does_not_reach_a_role_of_another_country() -> None:
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
db.executemany("INSERT INTO roles VALUES (?, ?, ?)", [
("Remote, Denmark", "remote denmark", "GB"), ("Copenhagen", "copenhagen", "DE"),
("Copenhagen", "copenhagen", "DK"), ("Nowhere, Denmark", "nowhere denmark", "")])
sql, params = match_sql("country:DK")
assert db.execute(f"SELECT country FROM roles WHERE {sql} ORDER BY country",
params).fetchall() == [("",), ("DK",)]
def test_country_eu_is_the_column_alone(table: sqlite3.Connection) -> None:
assert found(table, "country:EU") == {"Aarhus, Denmark; København, Denmark; Stockholm, Sweden",
"Copenhagen; London; Warsaw; Milano"}
def test_a_region_is_its_names_and_its_towns(table: sqlite3.Connection) -> None:
skane = found(table, "region:skane")
assert skane == {"Malmö", "Lund", "Sweden - Lund", "Helsingborg", "Höganäs, Sweden",
"Malmö, Skåne County, Sweden", "Skåne län, Sweden"}
assert found(table, "region:Skåne") == skane
hovedstaden = found(table, "region:hovedstaden")
assert {"København", "København K", "København Ø", "Copenhagen, Denmark", "Ballerup",
"Humlebæk, Denmark", "Kgs. Lyngby", "Kongens Lyngby", "Søborg", "Albertslund",
"Måløv, Capital Region of Denmark, Denmark", "Nordhavn og mulighed for hjemmearbejde",
} <= hovedstaden
assert not hovedstaden & {"Aarhus C, Denmark", "Billund, Denmark", "Malmö", "Nakskov"}
def test_zealand_is_the_danish_region_and_not_new_zealand(table: sqlite3.Connection) -> None:
assert found(table, "region:zealand") == {"Kalundborg, Region Zealand, Denmark", "Nakskov"}
def test_jutland_and_funen(table: sqlite3.Connection) -> None:
assert {"Billund, Denmark", "Aarhus C, Denmark"} <= found(table, "region:jutland")
assert found(table, "region:funen") == {"Odense, REGION_DNK 1852 South Denmark, Denmark"}
def test_oresund_is_the_two_capital_regions(table: sqlite3.Connection) -> None:
assert found(table, "region:oresund") == (
found(table, "region:hovedstaden") | found(table, "region:skane"))
def test_a_region_is_a_place_written_as_bare_text(table: sqlite3.Connection) -> None:
assert found(table, "Skåne") == found(table, "region:skane")
assert found(table, "place:scania") == found(table, "region:skane")
assert found(table, "Denmark") == found(table, "country:DK")
def test_a_token_that_names_nothing_finds_nothing(table: sqlite3.Connection) -> None:
assert found(table, "region:atlantis") == set()
assert found(table, "place:") == set()
assert found(table, "") == set()
assert found(table, "place:nowhere-in-particular") == set()
def test_the_columns_are_the_callers(table: sqlite3.Connection) -> None:
sql, params = match_sql("country:DK", place="r.pf", country="r.c")
assert "r.pf" in sql and "r.c " in sql and "place_fold" not in sql
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE r (pf TEXT, c TEXT)")
db.executemany("INSERT INTO r VALUES (?, ?)", [("lund", "SE"), ("billund", "DK")])
sql, params = match_sql("place:lund", place="r.pf", country="r.c")
assert db.execute(f"SELECT pf FROM r WHERE {sql}", params).fetchall() == [("lund",)]
def test_sql_is_parenthesised_and_has_one_parameter_per_question_mark() -> None:
for token in ("country:DK", "region:oresund", "place:copenhagen", "place:denmark",
"region:atlantis"):
sql, params = match_sql(token)
assert sql.startswith("(") and sql.endswith(")")
assert sql.count("?") == len(params), token
def test_a_wildcard_in_a_value_is_a_letter() -> None:
assert places._escape("50%_off\\") == "50\\%\\_off\\\\"
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
db.executemany("INSERT INTO roles VALUES (?, ?, '')", [("a b", "a b"), ("axb", "axb"),
("a_b", "a_b")])
for token in ("place:a_b", "place:a%b"):
sql, params = match_sql(token)
assert {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)} \
== {"a b"}, token
# A column that is not folded still matches a literal underscore, not any character.
sql, params = places._words("place", ["a_b"])
assert db.execute(f"SELECT place FROM roles WHERE {sql}", params).fetchall() == [("a_b",)]
# ----------------------------------------------------------------- the tables
def test_every_region_has_names_and_towns() -> None:
assert set(REGIONS) == {"skane", "hovedstaden", "zealand", "oresund", "jutland", "funen"}
for key, region in REGIONS.items():
assert region.key == key and region.label and region.names and region.towns
def test_the_bare_word_zealand_is_no_name_since_new_zealand_has_it() -> None:
assert "zealand" not in places._PHRASES["zealand"]
def test_the_capital_and_scanian_towns_are_in_their_regions() -> None:
skane = {fold(t) for t in REGIONS["skane"].towns}
assert {"malmo", "lund", "helsingborg", "kristianstad", "landskrona", "trelleborg",
"angelholm", "hassleholm", "ystad", "eslov", "hoganas", "staffanstorp", "lomma",
"bjuv", "burlov", "kavlinge", "svedala", "vellinge"} <= skane
hoved = {fold(t) for t in REGIONS["hovedstaden"].towns}
assert {"kobenhavn", "kgs lyngby", "soborg", "ballerup", "malov", "bagsvaerd", "humlebaek",
"smorum", "nordhavn", "frederiksberg", "hillerod", "helsingor"} <= hoved
# ----------------------------------------------------------------- suggestions
PLACES = [
("København", "DK", 69), ("Copenhagen, Denmark", "DK", 56), ("København K", "DK", 35),
("København Ø", "DK", 34), ("Kgs. Lyngby", "DK", 16), ("Billund, Denmark", "DK", 18),
("Aarhus N, Region Central Jutland, Denmark", "DK", 26), ("Albertslund", "DK", 10),
("Denmark", "DK", 27), ("Remote OR Copenhagen", "DK", 1),
("Malmö", "SE", 214), ("Lund", "SE", 127), ("Sweden - Lund", "SE", 29),
("Malmö, Skåne County, Sweden", "SE", 3), ("Stockholm, Sweden", "SE", 287),
("Aarhus, Denmark; København, Denmark; Stockholm, Sweden", "EU", 1),
("Remote", "", 40), ("Hybrid", "", 12), ("Europe", "EU", 20), ("", "", 5),
("Berlin, Germany", "DE", 100), ("Auckland, New Zealand", "NZ", 25),
("The Hague, Netherlands", "NL", 9), ("Den Haag", "NL", 4),
]
def tokens(q: str, **kw) -> list[str]:
return [s.token for s in suggest(q, PLACES, **kw)]
def test_skane_is_suggested_however_it_is_typed() -> None:
for q in ("Skåne", "skane", "SKANE", "ska", "Scania", "scan"):
assert "region:skane" in tokens(q), q
hit = next(s for s in suggest("skane", PLACES) if s.token == "region:skane")
assert hit == Suggestion("region:skane", "Skåne", "region", 214 + 127 + 29 + 3)
def test_a_country_counts_the_roles_that_carry_its_code() -> None:
hit = suggest("den", PLACES)[0]
assert hit == Suggestion("country:DK", "Denmark", "country",
69 + 56 + 35 + 34 + 16 + 18 + 26 + 10 + 27 + 1)
assert tokens("swe")[0] == "country:SE"
assert tokens("sverige")[0] == "country:SE"
assert tokens("danmark")[0] == "country:DK"
def test_countries_come_first_then_regions_then_towns() -> None:
kinds = [s.kind for s in suggest("den", PLACES)]
assert kinds == sorted(kinds, key=["country", "region", "place"].index)
assert kinds[0] == "country" and "region" in kinds and kinds[-1] == "place"
def test_towns_are_grouped_and_labelled_by_their_commonest_spelling() -> None:
lund = [s for s in suggest("lund", PLACES) if s.kind == "place"]
assert lund == [Suggestion("place:lund", "Lund", "place", 127 + 29)]
kbh = [s for s in suggest("kob", PLACES) if s.kind == "place"]
assert kbh[0].label == "København" and kbh[0].token == "place:kobenhavn"
def test_a_postal_district_is_not_a_town_of_its_own() -> None:
labels = [s.label for s in suggest("kob", PLACES) if s.kind == "place"]
assert "København K" not in labels and "København Ø" not in labels
assert [s.label for s in suggest("aarhus", PLACES) if s.kind == "place"] == ["Aarhus"]
def test_spellings_of_one_town_are_one_suggestion() -> None:
towns = [s for s in suggest("cop", PLACES) if s.kind == "place"]
assert len(towns) == 1
# København, K and Ø, the one in a list of three towns; Copenhagen, and "Remote OR Copenhagen".
assert towns[0].label == "København" and towns[0].count == 69 + 35 + 34 + 1 + 56 + 1
assert [s.token for s in suggest("kobenhavn", PLACES) if s.kind == "place"] \
== [towns[0].token]
def test_only_the_first_comma_part_of_each_semicolon_part_is_a_town() -> None:
labels = {s.label for q in "abcdefghijklmnopqrstuvwxyz"
for s in suggest(q, PLACES, limit=999) if s.kind == "place"}
assert {"Aarhus", "Stockholm", "Malmö", "Lund", "Berlin"} <= labels
assert not {"Denmark", "Sweden", "Region Central Jutland", "Remote", "Hybrid", "Europe"} \
& labels
def test_remote_hybrid_and_country_names_are_no_town() -> None:
for q in ("rem", "hyb", "eur", "denm"):
assert [s for s in suggest(q, PLACES) if s.kind == "place"] == [], q
def test_a_word_prefix_finds_a_multi_word_town() -> None:
assert tokens("haag") == tokens("hague") == tokens("den haag") == tokens("the hag") \
== ["place:the hague"]
assert tokens("kgs ly") == ["place:kgs lyngby"]
assert tokens("lyngby") == ["place:kgs lyngby"]
def test_the_word_must_start_not_end() -> None:
assert "place:lund" not in tokens("und")
assert "place:billund" not in tokens("lund")
assert "place:albertslund" not in tokens("lund")
def test_new_zealand_is_a_country_and_zealand_a_region_only_when_it_has_roles() -> None:
assert tokens("zea")[:1] == ["country:NZ"]
assert "region:zealand" not in tokens("zea") # no Zealand role in PLACES
assert "country:AU" not in tokens("aus") # no role of its own
def test_limit_and_the_empty_query() -> None:
assert suggest("", PLACES) == [] and suggest(" ", PLACES) == [] and suggest("---", PLACES) == []
assert len(suggest("a", PLACES, limit=3)) == 3
assert len(suggest("a", PLACES)) <= 12
def test_towns_are_ordered_by_roles() -> None:
towns = [s for s in suggest("l", PLACES) if s.kind == "place"]
assert [s.count for s in towns] == sorted((s.count for s in towns), reverse=True)
def test_suggest_takes_any_iterable() -> None:
assert suggest("lund", iter(PLACES)) == suggest("lund", PLACES)
def test_a_suggested_token_finds_the_roles_it_counted(table: sqlite3.Connection) -> None:
for s in suggest("kob", PLACES) + suggest("lund", PLACES) + suggest("skane", PLACES):
assert parse(s.token).kind == s.kind
assert found(table, s.token), s.token
assert label(s.token) == s.label or s.kind == "place"