Add places: folded whole-word place matching, country, region and town tokens, suggestions
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
64f367cc18
commit
ebb1f560de
2 files changed
+990
No files matched your search
+537
@@ -0,0 +1,537 @@
|
||||
"""Places as a filter: what a board writes in `place`, matched by whole words.
|
||||
|
||||
A location line is free text: "København K", "Kgs. Lyngby", "Aarhus N, Region
|
||||
Central Jutland, Denmark". A substring match finds `Lund` in `Billund` and
|
||||
`Denmark` in nothing that only names a town. Here a place is folded (casefolded,
|
||||
marks stripped, `ø` to `o`, `æ` to `ae`, `å` to `a`, punctuation to spaces) and
|
||||
matched as whole words, so `Lund` is `lund` and no other word. A filter is one
|
||||
of three tokens, and several tokens are joined by OR:
|
||||
|
||||
country:DK the role's country, or a line that names it while `country`
|
||||
is empty or `EU` (a board that lists several offices)
|
||||
region:skane any town or name of a region in `REGIONS`
|
||||
place:malmo the town's fold and its aliases, so `copenhagen`,
|
||||
`kobenhavn` and `kopenhamn` find each other
|
||||
|
||||
`match_sql` turns a token into SQL over `roles.place_fold` and `roles.country`;
|
||||
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
|
||||
chip's text back from the address. No database access here, the caller hands
|
||||
over the places it holds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
|
||||
|
||||
__all__ = ["REGIONS", "Region", "Suggestion", "Token", "fold", "label", "match_sql",
|
||||
"parse", "suggest"]
|
||||
|
||||
|
||||
def fold(text: str) -> str:
|
||||
"""`Skåne län` -> `skane lan`, `København K` -> `kobenhavn k`, `Kgs. Lyngby`
|
||||
-> `kgs lyngby`. What `roles.place_fold` holds. `å` needs no rule of its own:
|
||||
NFKD splits it into `a` and a ring, and the ring is a mark."""
|
||||
return _place_fold(text)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ countries
|
||||
|
||||
|
||||
def _countries() -> dict[str, tuple[str, ...]]:
|
||||
"""Every code to the names `_COUNTRY_NAMES` gives it, English name first."""
|
||||
found: dict[str, tuple[str, ...]] = {}
|
||||
for line in _COUNTRY_NAMES.split("\n"):
|
||||
if line:
|
||||
code, rest = line.split(" ", 1)
|
||||
found[code] = tuple(name.strip() for name in rest.split("|"))
|
||||
return found
|
||||
|
||||
|
||||
_COUNTRY_LABELS = _countries()
|
||||
|
||||
#: A code to the folded names a country goes by.
|
||||
_COUNTRY_FOLDS: dict[str, tuple[str, ...]] = {
|
||||
code: tuple(dict.fromkeys(filter(None, map(fold, names))))
|
||||
for code, names in _COUNTRY_LABELS.items()}
|
||||
|
||||
#: What a line has to say for a role of that country whose `country` column is
|
||||
#: empty or `EU`: a name of the country, or a city `country_of` knows in it. A
|
||||
#: board that lists offices ("Copenhagen; London; Warsaw") names cities only.
|
||||
_COUNTRY_MENTIONS: dict[str, tuple[str, ...]] = {
|
||||
code: tuple(dict.fromkeys((*names, *(fold(c) for c, cc in _CITIES.items() if cc == code))))
|
||||
for code, names in _COUNTRY_FOLDS.items()}
|
||||
|
||||
#: A folded country name to its code.
|
||||
_COUNTRY_BY_NAME: dict[str, str] = {
|
||||
name: code for code, names in _COUNTRY_FOLDS.items() for name in names}
|
||||
|
||||
#: Words that are not a town: a country, a macro region, how the role is worked.
|
||||
_NOT_A_TOWN = frozenset(_COUNTRY_BY_NAME) | frozenset(map(fold, _REGIONS)) \
|
||||
| {"remote", "hybrid"}
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- aliases
|
||||
|
||||
#: Towns boards write in more than one language, first spelling shown. Each
|
||||
#: group is symmetric: any spelling finds the others. The Nordics and the DACH
|
||||
#: countries in full for their large cities, and the European cities that
|
||||
#: appear both ways in boards.
|
||||
_ALIAS_GROUPS: tuple[tuple[str, ...], ...] = (
|
||||
# Denmark
|
||||
("Copenhagen", "København", "Kobenhavn", "Köpenhamn", "Kopenhamn", "Kopenhagen",
|
||||
"Copenhague", "Kopenhaga"),
|
||||
("Aarhus", "Århus"),
|
||||
("Aalborg", "Ålborg"),
|
||||
("Kgs. Lyngby", "Kongens Lyngby", "Lyngby"),
|
||||
("Taastrup", "Tåstrup"),
|
||||
("Elsinore", "Helsingør"),
|
||||
# Sweden
|
||||
("Gothenburg", "Göteborg", "Goteborg"),
|
||||
("Stockholm",),
|
||||
("Malmö", "Malmo"),
|
||||
("Helsingborg", "Hälsingborg"),
|
||||
("Västerås", "Vasteras"),
|
||||
("Linköping", "Linkoping"),
|
||||
("Örebro", "Orebro"),
|
||||
("Norrköping", "Norrkoping"),
|
||||
("Jönköping", "Jonkoping"),
|
||||
# Norway, Finland, Iceland
|
||||
("Oslo",),
|
||||
("Bergen",),
|
||||
("Trondheim", "Trondhjem"),
|
||||
("Helsinki", "Helsingfors"),
|
||||
("Espoo", "Esbo"),
|
||||
("Vantaa", "Vanda"),
|
||||
("Reykjavik", "Reykjavík"),
|
||||
# Germany
|
||||
("Munich", "München", "Muenchen", "Munchen"),
|
||||
("Cologne", "Köln", "Koeln", "Koln"),
|
||||
("Nuremberg", "Nürnberg", "Nuernberg", "Nurnberg"),
|
||||
("Düsseldorf", "Duesseldorf", "Dusseldorf"),
|
||||
("Hanover", "Hannover"),
|
||||
("Mainz", "Mayence"),
|
||||
("Frankfurt", "Frankfurt am Main"),
|
||||
("Göttingen", "Goettingen", "Gottingen"),
|
||||
("Würzburg", "Wuerzburg", "Wurzburg"),
|
||||
("Tübingen", "Tuebingen", "Tubingen"),
|
||||
("Lübeck", "Luebeck", "Lubeck"),
|
||||
("Münster", "Muenster", "Munster"),
|
||||
# Austria
|
||||
("Vienna", "Wien", "Vienne"),
|
||||
("Salzburg",),
|
||||
("Innsbruck",),
|
||||
# Switzerland
|
||||
("Zurich", "Zürich", "Zuerich"),
|
||||
("Geneva", "Genève", "Geneve", "Genf", "Ginevra"),
|
||||
("Basel", "Basle", "Bâle"),
|
||||
("Bern", "Berne"),
|
||||
("Lucerne", "Luzern"),
|
||||
("St. Gallen", "Sankt Gallen", "Saint Gall"),
|
||||
# The rest of Europe
|
||||
("Prague", "Praha", "Prag"),
|
||||
("Warsaw", "Warszawa", "Warschau"),
|
||||
("Krakow", "Kraków", "Cracow"),
|
||||
("Wroclaw", "Wrocław", "Breslau"),
|
||||
("Gdansk", "Gdańsk", "Danzig"),
|
||||
("Poznan", "Poznań", "Posen"),
|
||||
("Lisbon", "Lisboa", "Lissabon"),
|
||||
("Milan", "Milano", "Mailand"),
|
||||
("Rome", "Roma", "Rom"),
|
||||
("Turin", "Torino"),
|
||||
("Florence", "Firenze"),
|
||||
("Naples", "Napoli", "Neapel"),
|
||||
("Venice", "Venezia", "Venedig"),
|
||||
("Genoa", "Genova"),
|
||||
("Brussels", "Bruxelles", "Brussel", "Brüssel"),
|
||||
("Antwerp", "Antwerpen", "Anvers"),
|
||||
("Ghent", "Gent", "Gand"),
|
||||
("Bruges", "Brugge"),
|
||||
("Liège", "Liege", "Luik", "Lüttich"),
|
||||
("The Hague", "Den Haag", "s-Gravenhage"),
|
||||
("Luxembourg", "Luxemburg"),
|
||||
("Seville", "Sevilla"),
|
||||
("Athens", "Athina", "Athen"),
|
||||
("Bucharest", "București", "Bucuresti"),
|
||||
("Belgrade", "Beograd"),
|
||||
("Bratislava", "Pressburg"),
|
||||
("Kyiv", "Kiev"),
|
||||
("Cluj-Napoca", "Cluj"),
|
||||
("Timișoara", "Timisoara"),
|
||||
("Marseille", "Marseilles"),
|
||||
("Bolzano", "Bozen"),
|
||||
)
|
||||
|
||||
|
||||
def _variants(text: str) -> tuple[str, ...]:
|
||||
"""A fold and its `aa` spelling: Danish writes `å` as `aa` (`Aarhus`, `Taastrup`)
|
||||
and `Århus` and `Tåstrup` fold to `arhus` and `tastrup`."""
|
||||
return tuple(dict.fromkeys((text, text.replace("aa", "a"))))
|
||||
|
||||
|
||||
def _aliases() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]:
|
||||
"""Every folded spelling to its whole group (all folded), and to its shown
|
||||
spelling. The group starts with the first spelling listed."""
|
||||
groups: dict[str, tuple[str, ...]] = {}
|
||||
shown: dict[str, str] = {}
|
||||
for group in _ALIAS_GROUPS:
|
||||
members = tuple(dict.fromkeys(v for name in group for v in _variants(fold(name))))
|
||||
for name in group:
|
||||
shown.setdefault(fold(name), name)
|
||||
for member in members:
|
||||
groups[member] = members
|
||||
return groups, shown
|
||||
|
||||
|
||||
_GROUPS, _SHOWN = _aliases()
|
||||
|
||||
|
||||
def _expand(text: str) -> tuple[str, ...]:
|
||||
"""A folded town and everything it is also written as."""
|
||||
return _GROUPS.get(text) or _variants(text)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- regions
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Region:
|
||||
"""A region below a country, or across two. `names` are what a board calls
|
||||
the region itself, `towns` the places in it; a line is in the region when it
|
||||
has one of either as whole words. Spellings are shown with their marks and
|
||||
folded when used."""
|
||||
|
||||
key: str
|
||||
label: str
|
||||
names: tuple[str, ...]
|
||||
towns: tuple[str, ...]
|
||||
country: str = ""
|
||||
|
||||
|
||||
def _split(text: str) -> tuple[str, ...]:
|
||||
return tuple(part.strip() for part in text.split(",") if part.strip())
|
||||
|
||||
|
||||
_SKANE = Region(
|
||||
"skane", "Skåne",
|
||||
_split("Skåne, Skåne län, Skåne County, Region Skåne, Skåne Region, Scania, Skaane"),
|
||||
_split("""Malmö, Lund, Helsingborg, Kristianstad, Landskrona, Trelleborg, Ängelholm,
|
||||
Hässleholm, Ystad, Eslöv, Höganäs, Staffanstorp, Lomma, Bjuv, Burlöv, Kävlinge,
|
||||
Svedala, Vellinge, Simrishamn, Tomelilla, Sjöbo, Svalöv, Klippan, Åstorp,
|
||||
Bromölla, Osby, Perstorp, Örkelljunga, Östra Göinge, Hörby, Höör, Båstad,
|
||||
Skurup, Arlöv, Limhamn, Dalby, Åkarp, Bjärred, Höllviken, Ljungbyhed,
|
||||
Löddeköpinge, Furulund, Genarp, Veberöd, Skanör, Falsterbo, Mölle, Arild"""),
|
||||
"SE")
|
||||
|
||||
_HOVEDSTADEN = Region(
|
||||
"hovedstaden", "Capital Region of Denmark",
|
||||
_split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"),
|
||||
_split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup,
|
||||
Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte,
|
||||
Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre,
|
||||
Høje-Taastrup, Hørsholm, Ishøj, Lyngby-Taarbæk, Rudersdal, Rødovre, Tårnby,
|
||||
Vallensbæk, Kgs. Lyngby, Kongens Lyngby, Lyngby, Virum, Holte, Nærum, Birkerød,
|
||||
Vedbæk, Skodsborg, Klampenborg, Charlottenlund, Hellerup, Ordrup, Bagsværd,
|
||||
Søborg, Mørkhøj, Måløv, Skovlunde, Smørum, Ølstykke, Stenløse, Værløse, Farum,
|
||||
Slangerup, Hedehusene, Taastrup, Kastrup, Amager, Valby, Vanløse, Brønshøj,
|
||||
Nørrebro, Østerbro, Vesterbro, Nordhavn, Ørestad, Sydhavn, Islands Brygge,
|
||||
Refshaleøen, Humlebæk, Kokkedal, Espergærde, Snekkersten, Rungsted, Rungsted Kyst,
|
||||
Lillerød, Frederiksværk, Hundested, Helsinge, Gilleleje, Græsted, Vallensbæk Strand,
|
||||
Skævinge, Rønne, Nexø, Allinge, Svaneke, Aakirkeby, Hasle"""),
|
||||
"DK")
|
||||
|
||||
# Region Sjælland, not the island: Copenhagen is on Zealand too, but a board that
|
||||
# writes "Region Zealand" means the region and the Capital Region has its own key.
|
||||
# The bare word is left out of `names` since "New Zealand" would match it.
|
||||
_ZEALAND = Region(
|
||||
"zealand", "Zealand",
|
||||
_split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"),
|
||||
_split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved,
|
||||
Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg,
|
||||
Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing,
|
||||
Haslev, Korsør, Skælskør, Præstø, Stege, Fakse, Rødby, Rødbyhavn, Jyllinge,
|
||||
Viby Sjælland, Solrød Strand, Fuglebjerg, Gørlev, Dianalund, Jyderup, Hvalsø,
|
||||
Tølløse, Hørve, Fårevejle, Bjæverskov, Nørre Asmindrup"""),
|
||||
"DK")
|
||||
|
||||
_JUTLAND = Region(
|
||||
"jutland", "Jutland",
|
||||
_split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland,
|
||||
North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland,
|
||||
East Jutland, West Jutland, Vestjylland, Østjylland"""),
|
||||
_split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning,
|
||||
Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov,
|
||||
Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå,
|
||||
Hjørring, Frederikshavn, Skagen, Thisted, Struer, Ikast, Brande, Varde, Ribe,
|
||||
Tønder, Vojens, Hadsten, Hobro, Skanderborg, Odder, Hedensted, Vejen, Ebeltoft,
|
||||
Lemvig, Rønde, Egå, Lystrup, Hinnerup, Tilst, Hammel, Galten, Sabro, Harlev,
|
||||
Løsning, Højbjerg, Juelsminde, Nørresundby, Brønderslev, Aars, Nibe, Sæby, Løgstør, Tarm,
|
||||
Skjern, Padborg, Rødekro, Toftlund, Bramming, Kruså, Brabrand, Ry, Gjern"""),
|
||||
"DK")
|
||||
|
||||
_FUNEN = Region(
|
||||
"funen", "Funen",
|
||||
_split("Funen, Fyn, Region Fyn"),
|
||||
_split("""Odense, Svendborg, Nyborg, Faaborg, Fåborg, Middelfart, Assens, Kerteminde,
|
||||
Bogense, Langeskov, Ringe, Otterup, Munkebo, Vissenbjerg, Årslev, Tommerup,
|
||||
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
|
||||
"DK")
|
||||
|
||||
# The Øresund region is the two capital regions either side of the sound.
|
||||
_ORESUND = Region(
|
||||
"oresund", "Øresund",
|
||||
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen")
|
||||
+ _HOVEDSTADEN.names + _SKANE.names,
|
||||
_HOVEDSTADEN.towns + _SKANE.towns)
|
||||
|
||||
REGIONS: dict[str, Region] = {
|
||||
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)}
|
||||
|
||||
|
||||
def _phrases(region: Region) -> tuple[str, ...]:
|
||||
"""Every folded phrase that puts a line in the region, with the other
|
||||
spellings of its towns."""
|
||||
found: dict[str, None] = {}
|
||||
for text in (*region.names, *region.towns):
|
||||
for form in _expand(fold(text)):
|
||||
found[form] = None
|
||||
return tuple(found)
|
||||
|
||||
|
||||
_PHRASES: dict[str, tuple[str, ...]] = {key: _phrases(r) for key, r in REGIONS.items()}
|
||||
|
||||
#: A folded key, label or name to the region key; a name in two regions ("Capital
|
||||
#: Region of Denmark" is also in Øresund) belongs to the one that lists it first.
|
||||
_REGION_BY_NAME: dict[str, str] = {}
|
||||
for _key, _region in REGIONS.items():
|
||||
for _name in (_region.key, _region.label, *_region.names):
|
||||
_REGION_BY_NAME.setdefault(fold(_name), _key)
|
||||
|
||||
#: What a place is shown as when a chip is read from the address: every alias
|
||||
#: spelling, every region and every town of one, as written above.
|
||||
_TOWN_LABELS: dict[str, str] = dict(_SHOWN)
|
||||
for _region in REGIONS.values():
|
||||
for _town in (_region.label, *_region.towns, *_region.names):
|
||||
_TOWN_LABELS.setdefault(fold(_town), _town)
|
||||
|
||||
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- tokens
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Token:
|
||||
"""One place filter. `value` is a country code, a region key or a fold."""
|
||||
|
||||
kind: str
|
||||
value: str
|
||||
label: str
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.kind}:{self.value}"
|
||||
|
||||
|
||||
def _region_key(text: str) -> str:
|
||||
folded = fold(text)
|
||||
return _REGION_BY_NAME.get(folded, folded)
|
||||
|
||||
|
||||
def parse(token: str) -> Token:
|
||||
"""`country:DK`, `region:skane` or `place:malmo`. Bare text, or a prefix that
|
||||
is none of these, is a place: `Skåne` is `place:skane`. A country by name
|
||||
(`country:Denmark`) and a region by label (`region:Skåne`) are read as well."""
|
||||
kind, sep, value = (token or "").partition(":")
|
||||
kind = kind.strip().lower()
|
||||
if not sep or kind not in ("country", "region", "place"):
|
||||
kind, value = "place", token or ""
|
||||
value = value.strip()
|
||||
if kind == "country":
|
||||
code = value.upper()
|
||||
if code not in _COUNTRY_LABELS:
|
||||
code = _COUNTRY_BY_NAME.get(fold(value), code)
|
||||
return Token("country", code, _COUNTRY_LABELS.get(code, (code,))[0])
|
||||
if kind == "region":
|
||||
key = _region_key(value)
|
||||
return Token("region", key, REGIONS[key].label if key in REGIONS else key)
|
||||
folded = fold(value)
|
||||
return Token("place", folded, _TOWN_LABELS.get(folded) or folded.title())
|
||||
|
||||
|
||||
def label(token: str | Token) -> str:
|
||||
"""The text of a chip read from the address: `country:DK` is `Denmark`,
|
||||
`region:skane` is `Skåne`, `place:malmo` is `Malmö`. A town the tables do not
|
||||
know is its fold in title case."""
|
||||
return (token if isinstance(token, Token) else parse(token)).label
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ SQL
|
||||
|
||||
_NEVER = ("0", [])
|
||||
|
||||
|
||||
def _escape(word: str) -> str:
|
||||
"""A folded word holds no wildcard, but a caller may hand over raw text."""
|
||||
return re.sub(r"([\\%_])", r"\\\1", word)
|
||||
|
||||
|
||||
def _words(column: str, phrases: Iterable[str]) -> tuple[str, list]:
|
||||
"""Any of the phrases as whole words of the column. A multi-word phrase is
|
||||
a phrase: `kgs lyngby` needs both words, side by side."""
|
||||
phrases = [p for p in dict.fromkeys(phrases) if p]
|
||||
if not phrases:
|
||||
return _NEVER
|
||||
clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'"
|
||||
return " OR ".join([clause] * len(phrases)), [f"% {_escape(p)} %" for p in phrases]
|
||||
|
||||
|
||||
def _country(code: str, place: str, country: str) -> tuple[str, list]:
|
||||
"""The role's own country, or a line that names the country while `country`
|
||||
is empty or `EU`: a board listing several offices has no single country."""
|
||||
names, params = _words(place, _COUNTRY_MENTIONS.get(code, ()))
|
||||
if not params:
|
||||
return f"{country} = ?", [code]
|
||||
return f"{country} = ? OR ({country} IN ('', 'EU') AND ({names}))", [code, *params]
|
||||
|
||||
|
||||
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
|
||||
-> tuple[str, list]:
|
||||
"""SQL over the two columns for one token, parenthesised, with `?` parameters.
|
||||
Several tokens are joined with OR by the caller. Bare text is a place.
|
||||
|
||||
A `place:` value that is also a region or a country is matched as that too:
|
||||
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
|
||||
written as a bare word means what the owner meant by it. A token that names
|
||||
nothing known (`region:atlantis`, an empty value) matches no role."""
|
||||
tok = parse(token)
|
||||
if tok.kind == "country":
|
||||
sql, params = _country(tok.value, place, country)
|
||||
elif tok.kind == "region":
|
||||
sql, params = _words(place, _PHRASES.get(tok.value, ()))
|
||||
else:
|
||||
region = _REGION_BY_NAME.get(tok.value)
|
||||
sql, params = _words(place, [*_expand(tok.value), *_PHRASES.get(region, ())]) \
|
||||
if tok.value else _NEVER
|
||||
if tok.value in _COUNTRY_BY_NAME:
|
||||
more, extra = _country(_COUNTRY_BY_NAME[tok.value], place, country)
|
||||
sql, params = f"{sql} OR {more}", [*params, *extra]
|
||||
return f"({sql})", params
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- suggestions
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Suggestion:
|
||||
"""`token` is what the chip holds, `kind` is `country`, `region` or `place`,
|
||||
`count` the open roles it stands for."""
|
||||
|
||||
token: str
|
||||
label: str
|
||||
kind: str
|
||||
count: int
|
||||
|
||||
|
||||
def _prefixes(q: str, candidates: Iterable[str]) -> bool:
|
||||
"""`q` starts a word of a candidate, or a run of words: `kgs ly` finds
|
||||
`kgs lyngby`, `hague` finds `the hague`."""
|
||||
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
|
||||
|
||||
|
||||
#: Where one town ends in a line: `;` between places, and `or` / `eller` / `og`
|
||||
#: between two towns in one ("København V eller Aalborg").
|
||||
_PART = re.compile(r"\s*;\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
|
||||
|
||||
#: Danish boards append "and the possibility of working from home".
|
||||
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
|
||||
|
||||
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
|
||||
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
|
||||
|
||||
#: A country or "Remote" ahead of the town, "Sweden - Lund", "Remote-New Zealand".
|
||||
_LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$")
|
||||
|
||||
|
||||
def _towns(place: str) -> list[str]:
|
||||
"""The first comma part of each place of a line: `Aarhus N, Region Central
|
||||
Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal
|
||||
district is dropped, since `place:kobenhavn` finds `København K` anyway."""
|
||||
found = []
|
||||
for part in _PART.split(_AND_HOME.sub("", place or "")):
|
||||
head = part.split(",")[0].strip()
|
||||
if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN:
|
||||
head = lead[2]
|
||||
head = _DISTRICT.sub("", head)
|
||||
if fold(head) and fold(head) not in _NOT_A_TOWN:
|
||||
found.append(head)
|
||||
return found
|
||||
|
||||
|
||||
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
|
||||
"""Open roles per region, over folded lines. A line is cut into every run of
|
||||
words once, and each region asks whether it holds one of its phrases."""
|
||||
sets = {key: set(_PHRASES[key]) for key in keys}
|
||||
longest = max((len(p.split()) for key in sets for p in sets[key]), default=1)
|
||||
counts: Counter[str] = Counter()
|
||||
for text, roles in places:
|
||||
words = text.split()
|
||||
runs = {" ".join(words[i:i + n])
|
||||
for n in range(1, longest + 1) for i in range(len(words) - n + 1)}
|
||||
for key, phrases in sets.items():
|
||||
if not phrases.isdisjoint(runs):
|
||||
counts[key] += roles
|
||||
return counts
|
||||
|
||||
|
||||
def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
|
||||
-> list[Suggestion]:
|
||||
"""Tokens for the text typed so far. `places` is `(place, country, open
|
||||
roles)`. Countries first, then regions, then towns, each by open roles.
|
||||
|
||||
A country is suggested when `q` starts a word of one of its names, with the
|
||||
roles that carry its code. A region likewise on its label and names, with the
|
||||
roles whose line is in it. A town is the first comma part of each `;` part of
|
||||
a line, grouped by fold and by alias (`Copenhagen` and `København` are one
|
||||
town, shown as its commonest spelling). Countries, regions and towns with no
|
||||
open role are left out: a chip that finds nothing helps nobody."""
|
||||
text = fold(q)
|
||||
if not text:
|
||||
return []
|
||||
rows = [(place or "", country or "", int(roles)) for place, country, roles in places]
|
||||
|
||||
by_country: Counter[str] = Counter()
|
||||
for _, country, roles in rows:
|
||||
by_country[country] += roles
|
||||
countries = [Suggestion(f"country:{code}", names[0], "country", by_country[code])
|
||||
for code, names in _COUNTRY_LABELS.items()
|
||||
if by_country[code] and _prefixes(text, _COUNTRY_FOLDS[code])]
|
||||
|
||||
named = [key for key, r in REGIONS.items()
|
||||
if _prefixes(text, [fold(r.label), *map(fold, r.names)])]
|
||||
counts = _region_counts(named, [(fold(place), roles) for place, _, roles in rows])
|
||||
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
|
||||
for key in named if counts[key]]
|
||||
|
||||
spellings: dict[str, Counter[str]] = {}
|
||||
written: dict[str, set[str]] = {}
|
||||
for place, _, roles in rows:
|
||||
for town in _towns(place):
|
||||
folded = fold(town)
|
||||
group = _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
|
||||
spellings.setdefault(group, Counter())[town] += roles
|
||||
written.setdefault(group, set()).add(folded)
|
||||
towns = []
|
||||
for group, seen in spellings.items():
|
||||
if not _prefixes(text, [form for folded in written[group] for form in _expand(folded)]):
|
||||
continue
|
||||
shown = min(seen, key=lambda s: (-seen[s], s))
|
||||
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", sum(seen.values())))
|
||||
|
||||
countries.sort(key=lambda s: (-s.count, s.label))
|
||||
regions.sort(key=lambda s: (-s.count, s.label))
|
||||
towns.sort(key=lambda s: (-s.count, s.label))
|
||||
return (countries + regions + towns)[:limit]
|
||||
@@ -0,0 +1,453 @@
|
||||
"""`app.places`: folding, the three token kinds run as SQL, and the suggestions.
|
||||
|
||||
The rows are `place` strings as boards write them, taken from the live database.
|
||||
Each token's SQL runs against an in-memory table shaped like `roles`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from app import places
|
||||
from app.places import REGIONS, Suggestion, Token, fold, label, match_sql, parse, suggest
|
||||
|
||||
#: (place, country) as a board wrote them and `country_of` read them.
|
||||
ROWS = [
|
||||
("København", "DK"), ("Copenhagen, Denmark", "DK"), ("Humlebæk, Denmark", "DK"),
|
||||
("København K", "DK"), ("København Ø", "DK"), ("Ballerup", "DK"), ("Denmark", "DK"),
|
||||
("Aarhus N, Region Central Jutland, Denmark", "DK"), ("Aarhus C, Denmark", "DK"),
|
||||
("Billund, Denmark", "DK"), ("Kgs. Lyngby", "DK"), ("Kongens Lyngby", "DK"),
|
||||
("Måløv, Capital Region of Denmark, Denmark", "DK"), ("Søborg", "DK"),
|
||||
("Kalundborg, Region Zealand, Denmark", "DK"), ("Albertslund", "DK"),
|
||||
("Odense, REGION_DNK 1852 South Denmark, Denmark", "DK"), ("Nakskov", "DK"),
|
||||
("Nordhavn og mulighed for hjemmearbejde", "DK"), ("Remote OR Copenhagen", "DK"),
|
||||
("Malmö", "SE"), ("Lund", "SE"), ("Sweden - Lund", "SE"), ("Helsingborg", "SE"),
|
||||
("Malmö, Skåne County, Sweden", "SE"), ("Skåne län, Sweden", "SE"),
|
||||
("Höganäs, Sweden", "SE"), ("Stockholm, Sweden", "SE"), ("Gothenburg, Sweden", "SE"),
|
||||
("Aarhus, Denmark; København, Denmark; Stockholm, Sweden", "EU"),
|
||||
("Copenhagen; London; Warsaw; Milano", "EU"),
|
||||
("Copenhagen; Warsaw; New York", ""),
|
||||
("Berlin, Germany; Munich, Germany", "DE"), ("Berlin, Germany", "DE"),
|
||||
("München", "DE"), ("Munich, Bavaria, Germany", "DE"), ("Köln", "DE"), ("Cologne", "DE"),
|
||||
("Zürich", "CH"), ("Zurich, Switzerland", "CH"), ("Genève", "CH"), ("Geneva", "CH"),
|
||||
("Wien", "AT"), ("Vienna, Austria", "AT"),
|
||||
("Auckland, New Zealand", "NZ"), ("Remote", ""), ("Hybrid", ""), ("Europe", ""),
|
||||
("Remote - Denmark", "DK"), ("", ""),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def table() -> sqlite3.Connection:
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, ?)",
|
||||
[(place, fold(place), country) for place, country in ROWS])
|
||||
return db
|
||||
|
||||
|
||||
def found(db: sqlite3.Connection, *tokens: str) -> set[str]:
|
||||
"""The places the tokens find, joined with OR as a caller would."""
|
||||
parts = [match_sql(token) for token in tokens]
|
||||
sql = " OR ".join(sql for sql, _ in parts)
|
||||
params = [p for _, params in parts for p in params]
|
||||
return {row[0] for row in db.execute(f"SELECT place FROM roles WHERE {sql}", params)}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- folding
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, folded", [
|
||||
("Skåne län", "skane lan"),
|
||||
("København K", "kobenhavn k"),
|
||||
("Kgs. Lyngby", "kgs lyngby"),
|
||||
("Malmö", "malmo"),
|
||||
("Åbyhøj", "abyhoj"),
|
||||
("Ærø", "aero"),
|
||||
("Düsseldorf", "dusseldorf"),
|
||||
("Zürich", "zurich"),
|
||||
("Wrocław", "wroclaw"),
|
||||
("Höje-Taastrup, Denmark", "hoje taastrup denmark"),
|
||||
(" Aarhus N ", "aarhus n"),
|
||||
("", ""),
|
||||
])
|
||||
def test_fold(text: str, folded: str) -> None:
|
||||
assert fold(text) == folded
|
||||
|
||||
|
||||
def test_fold_turns_a_ring_and_umlauts_into_the_plain_letter() -> None:
|
||||
assert fold("å") == "a"
|
||||
assert (fold("ä"), fold("ö"), fold("ü")) == ("a", "o", "u")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- parsing
|
||||
|
||||
|
||||
def test_parse_reads_the_three_kinds() -> None:
|
||||
assert parse("country:DK") == Token("country", "DK", "Denmark")
|
||||
assert parse("region:skane") == Token("region", "skane", "Skåne")
|
||||
assert parse("place:malmo") == Token("place", "malmo", "Malmö")
|
||||
|
||||
|
||||
def test_parse_reads_bare_text_as_a_place() -> None:
|
||||
assert parse("Skåne") == Token("place", "skane", "Skåne")
|
||||
assert parse("Kgs. Lyngby").value == "kgs lyngby"
|
||||
assert parse("Foo: Bar").kind == "place"
|
||||
|
||||
|
||||
def test_parse_is_forgiving_about_case_and_names() -> None:
|
||||
assert parse("Country:dk").value == "DK"
|
||||
assert parse("country:Denmark").value == "DK"
|
||||
assert parse("region:Skåne").value == "skane"
|
||||
assert parse("region:Capital Region of Denmark").value == "hovedstaden"
|
||||
assert parse("place:København").value == "kobenhavn"
|
||||
|
||||
|
||||
def test_a_token_round_trips_through_its_string() -> None:
|
||||
for text in ("country:DK", "region:oresund", "place:kgs lyngby"):
|
||||
assert str(parse(text)) == text
|
||||
|
||||
|
||||
def test_label_reads_a_chip_from_the_address() -> None:
|
||||
assert label("country:SE") == "Sweden"
|
||||
assert label("region:hovedstaden") == "Capital Region of Denmark"
|
||||
assert label("place:kobenhavn") == "København"
|
||||
assert label("place:copenhagen") == "Copenhagen"
|
||||
assert label("place:malmo") == "Malmö"
|
||||
assert label("place:tromso") == "Tromso"
|
||||
assert label(parse("place:kgs lyngby")) == "Kgs. Lyngby"
|
||||
assert label("country:ZZ") == "ZZ"
|
||||
assert label("region:atlantis") == "atlantis"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- SQL
|
||||
|
||||
|
||||
def test_a_place_is_matched_on_whole_words(table: sqlite3.Connection) -> None:
|
||||
lund = found(table, "place:lund")
|
||||
assert lund == {"Lund", "Sweden - Lund"}
|
||||
assert "Billund, Denmark" not in lund and "Albertslund" not in lund
|
||||
assert found(table, "place:billund") == {"Billund, Denmark"}
|
||||
|
||||
|
||||
def test_a_multi_word_place_is_a_phrase(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "place:kgs lyngby") == {"Kgs. Lyngby", "Kongens Lyngby"}
|
||||
assert found(table, "place:capital denmark") == set()
|
||||
assert found(table, "place:denmark capital") == set()
|
||||
|
||||
|
||||
def test_diacritics_and_spellings_meet(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "place:Skåne") == found(table, "place:skane")
|
||||
assert "Malmö" in found(table, "place:malmo") and "Malmö" in found(table, "place:Malmö")
|
||||
assert {"København", "København K"} <= found(table, "place:kobenhavn")
|
||||
|
||||
|
||||
COPENHAGENS = {"København", "Copenhagen, Denmark", "København K", "Remote OR Copenhagen"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spelling", ["copenhagen", "kobenhavn", "kopenhamn", "kopenhagen",
|
||||
"København", "Köpenhamn"])
|
||||
def test_copenhagen_is_every_spelling(table: sqlite3.Connection, spelling: str) -> None:
|
||||
assert found(table, f"place:{spelling}") >= COPENHAGENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group", places._ALIAS_GROUPS, ids=lambda g: g[0])
|
||||
def test_every_alias_finds_every_other(group: tuple[str, ...]) -> None:
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, '')",
|
||||
[(name, fold(name)) for name in group])
|
||||
for spelling in group:
|
||||
sql, params = match_sql(f"place:{spelling}")
|
||||
hit = {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)}
|
||||
assert hit == set(group), spelling
|
||||
|
||||
|
||||
def test_no_spelling_belongs_to_two_groups() -> None:
|
||||
seen: dict[str, str] = {}
|
||||
for group in places._ALIAS_GROUPS:
|
||||
for name in group:
|
||||
assert seen.setdefault(fold(name), group[0]) == group[0], name
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pair", [("Munich", "München"), ("Cologne", "Köln"),
|
||||
("Vienna", "Wien"), ("Zurich", "Zürich"),
|
||||
("Geneva", "Genève")])
|
||||
def test_a_city_written_two_ways_is_found_either_way(
|
||||
table: sqlite3.Connection, pair: tuple[str, str]) -> None:
|
||||
one, other = pair
|
||||
hit = found(table, f"place:{one}")
|
||||
assert hit == found(table, f"place:{other}")
|
||||
words = {w for place in hit for w in fold(place).split()}
|
||||
assert {fold(one), fold(other)} <= words
|
||||
|
||||
|
||||
def test_aa_and_the_ring_are_one_letter(table: sqlite3.Connection) -> None:
|
||||
hit = found(table, "place:århus")
|
||||
assert hit == found(table, "place:aarhus") == {
|
||||
"Aarhus N, Region Central Jutland, Denmark", "Aarhus C, Denmark",
|
||||
"Aarhus, Denmark; København, Denmark; Stockholm, Sweden"}
|
||||
|
||||
|
||||
def test_a_line_with_several_places_is_found_by_each(table: sqlite3.Connection) -> None:
|
||||
line = "Aarhus, Denmark; København, Denmark; Stockholm, Sweden"
|
||||
for tok in ("place:aarhus", "place:kobenhavn", "place:stockholm", "place:copenhagen"):
|
||||
assert line in found(table, tok), tok
|
||||
|
||||
|
||||
def test_several_tokens_are_joined_by_or(table: sqlite3.Connection) -> None:
|
||||
hit = found(table, "country:DK", "region:skane")
|
||||
assert {"København", "Malmö", "Lund"} <= hit
|
||||
assert "Stockholm, Sweden" not in hit
|
||||
|
||||
|
||||
def test_a_country_is_its_code(table: sqlite3.Connection) -> None:
|
||||
hit = found(table, "country:DK")
|
||||
assert {"København", "Denmark", "Ballerup", "Nakskov", "Odense, REGION_DNK 1852 South Denmark, Denmark"} <= hit
|
||||
assert "Malmö" not in hit and "Berlin, Germany" not in hit
|
||||
assert "Auckland, New Zealand" not in hit
|
||||
|
||||
|
||||
def test_a_country_is_also_a_line_that_names_it_when_country_is_unset(
|
||||
table: sqlite3.Connection) -> None:
|
||||
hit = found(table, "country:DK")
|
||||
# `country` is EU: a board listing offices in several countries.
|
||||
assert "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" in hit
|
||||
# Only cities are named, and one of them is Danish.
|
||||
assert "Copenhagen; London; Warsaw; Milano" in hit
|
||||
assert "Copenhagen; Warsaw; New York" in hit
|
||||
# The same words under a country that is set are that country's business.
|
||||
assert found(table, "country:SE") >= {"Stockholm, Sweden", "Malmö"}
|
||||
assert "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" in found(table, "country:SE")
|
||||
assert "Copenhagen, Denmark" in hit # its own country is DK
|
||||
|
||||
|
||||
def test_a_country_name_does_not_reach_a_role_of_another_country() -> None:
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, ?)", [
|
||||
("Remote, Denmark", "remote denmark", "GB"), ("Copenhagen", "copenhagen", "DE"),
|
||||
("Copenhagen", "copenhagen", "DK"), ("Nowhere, Denmark", "nowhere denmark", "")])
|
||||
sql, params = match_sql("country:DK")
|
||||
assert db.execute(f"SELECT country FROM roles WHERE {sql} ORDER BY country",
|
||||
params).fetchall() == [("",), ("DK",)]
|
||||
|
||||
|
||||
def test_country_eu_is_the_column_alone(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "country:EU") == {"Aarhus, Denmark; København, Denmark; Stockholm, Sweden",
|
||||
"Copenhagen; London; Warsaw; Milano"}
|
||||
|
||||
|
||||
def test_a_region_is_its_names_and_its_towns(table: sqlite3.Connection) -> None:
|
||||
skane = found(table, "region:skane")
|
||||
assert skane == {"Malmö", "Lund", "Sweden - Lund", "Helsingborg", "Höganäs, Sweden",
|
||||
"Malmö, Skåne County, Sweden", "Skåne län, Sweden"}
|
||||
assert found(table, "region:Skåne") == skane
|
||||
|
||||
hovedstaden = found(table, "region:hovedstaden")
|
||||
assert {"København", "København K", "København Ø", "Copenhagen, Denmark", "Ballerup",
|
||||
"Humlebæk, Denmark", "Kgs. Lyngby", "Kongens Lyngby", "Søborg", "Albertslund",
|
||||
"Måløv, Capital Region of Denmark, Denmark", "Nordhavn og mulighed for hjemmearbejde",
|
||||
} <= hovedstaden
|
||||
assert not hovedstaden & {"Aarhus C, Denmark", "Billund, Denmark", "Malmö", "Nakskov"}
|
||||
|
||||
|
||||
def test_zealand_is_the_danish_region_and_not_new_zealand(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "region:zealand") == {"Kalundborg, Region Zealand, Denmark", "Nakskov"}
|
||||
|
||||
|
||||
def test_jutland_and_funen(table: sqlite3.Connection) -> None:
|
||||
assert {"Billund, Denmark", "Aarhus C, Denmark"} <= found(table, "region:jutland")
|
||||
assert found(table, "region:funen") == {"Odense, REGION_DNK 1852 South Denmark, Denmark"}
|
||||
|
||||
|
||||
def test_oresund_is_the_two_capital_regions(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "region:oresund") == (
|
||||
found(table, "region:hovedstaden") | found(table, "region:skane"))
|
||||
|
||||
|
||||
def test_a_region_is_a_place_written_as_bare_text(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "Skåne") == found(table, "region:skane")
|
||||
assert found(table, "place:scania") == found(table, "region:skane")
|
||||
assert found(table, "Denmark") == found(table, "country:DK")
|
||||
|
||||
|
||||
def test_a_token_that_names_nothing_finds_nothing(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "region:atlantis") == set()
|
||||
assert found(table, "place:") == set()
|
||||
assert found(table, "") == set()
|
||||
assert found(table, "place:nowhere-in-particular") == set()
|
||||
|
||||
|
||||
def test_the_columns_are_the_callers(table: sqlite3.Connection) -> None:
|
||||
sql, params = match_sql("country:DK", place="r.pf", country="r.c")
|
||||
assert "r.pf" in sql and "r.c " in sql and "place_fold" not in sql
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE r (pf TEXT, c TEXT)")
|
||||
db.executemany("INSERT INTO r VALUES (?, ?)", [("lund", "SE"), ("billund", "DK")])
|
||||
sql, params = match_sql("place:lund", place="r.pf", country="r.c")
|
||||
assert db.execute(f"SELECT pf FROM r WHERE {sql}", params).fetchall() == [("lund",)]
|
||||
|
||||
|
||||
def test_sql_is_parenthesised_and_has_one_parameter_per_question_mark() -> None:
|
||||
for token in ("country:DK", "region:oresund", "place:copenhagen", "place:denmark",
|
||||
"region:atlantis"):
|
||||
sql, params = match_sql(token)
|
||||
assert sql.startswith("(") and sql.endswith(")")
|
||||
assert sql.count("?") == len(params), token
|
||||
|
||||
|
||||
def test_a_wildcard_in_a_value_is_a_letter() -> None:
|
||||
assert places._escape("50%_off\\") == "50\\%\\_off\\\\"
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, '')", [("a b", "a b"), ("axb", "axb"),
|
||||
("a_b", "a_b")])
|
||||
for token in ("place:a_b", "place:a%b"):
|
||||
sql, params = match_sql(token)
|
||||
assert {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)} \
|
||||
== {"a b"}, token
|
||||
# A column that is not folded still matches a literal underscore, not any character.
|
||||
sql, params = places._words("place", ["a_b"])
|
||||
assert db.execute(f"SELECT place FROM roles WHERE {sql}", params).fetchall() == [("a_b",)]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- the tables
|
||||
|
||||
|
||||
def test_every_region_has_names_and_towns() -> None:
|
||||
assert set(REGIONS) == {"skane", "hovedstaden", "zealand", "oresund", "jutland", "funen"}
|
||||
for key, region in REGIONS.items():
|
||||
assert region.key == key and region.label and region.names and region.towns
|
||||
|
||||
|
||||
def test_the_bare_word_zealand_is_no_name_since_new_zealand_has_it() -> None:
|
||||
assert "zealand" not in places._PHRASES["zealand"]
|
||||
|
||||
|
||||
def test_the_capital_and_scanian_towns_are_in_their_regions() -> None:
|
||||
skane = {fold(t) for t in REGIONS["skane"].towns}
|
||||
assert {"malmo", "lund", "helsingborg", "kristianstad", "landskrona", "trelleborg",
|
||||
"angelholm", "hassleholm", "ystad", "eslov", "hoganas", "staffanstorp", "lomma",
|
||||
"bjuv", "burlov", "kavlinge", "svedala", "vellinge"} <= skane
|
||||
hoved = {fold(t) for t in REGIONS["hovedstaden"].towns}
|
||||
assert {"kobenhavn", "kgs lyngby", "soborg", "ballerup", "malov", "bagsvaerd", "humlebaek",
|
||||
"smorum", "nordhavn", "frederiksberg", "hillerod", "helsingor"} <= hoved
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- suggestions
|
||||
|
||||
PLACES = [
|
||||
("København", "DK", 69), ("Copenhagen, Denmark", "DK", 56), ("København K", "DK", 35),
|
||||
("København Ø", "DK", 34), ("Kgs. Lyngby", "DK", 16), ("Billund, Denmark", "DK", 18),
|
||||
("Aarhus N, Region Central Jutland, Denmark", "DK", 26), ("Albertslund", "DK", 10),
|
||||
("Denmark", "DK", 27), ("Remote OR Copenhagen", "DK", 1),
|
||||
("Malmö", "SE", 214), ("Lund", "SE", 127), ("Sweden - Lund", "SE", 29),
|
||||
("Malmö, Skåne County, Sweden", "SE", 3), ("Stockholm, Sweden", "SE", 287),
|
||||
("Aarhus, Denmark; København, Denmark; Stockholm, Sweden", "EU", 1),
|
||||
("Remote", "", 40), ("Hybrid", "", 12), ("Europe", "EU", 20), ("", "", 5),
|
||||
("Berlin, Germany", "DE", 100), ("Auckland, New Zealand", "NZ", 25),
|
||||
("The Hague, Netherlands", "NL", 9), ("Den Haag", "NL", 4),
|
||||
]
|
||||
|
||||
|
||||
def tokens(q: str, **kw) -> list[str]:
|
||||
return [s.token for s in suggest(q, PLACES, **kw)]
|
||||
|
||||
|
||||
def test_skane_is_suggested_however_it_is_typed() -> None:
|
||||
for q in ("Skåne", "skane", "SKANE", "ska", "Scania", "scan"):
|
||||
assert "region:skane" in tokens(q), q
|
||||
hit = next(s for s in suggest("skane", PLACES) if s.token == "region:skane")
|
||||
assert hit == Suggestion("region:skane", "Skåne", "region", 214 + 127 + 29 + 3)
|
||||
|
||||
|
||||
def test_a_country_counts_the_roles_that_carry_its_code() -> None:
|
||||
hit = suggest("den", PLACES)[0]
|
||||
assert hit == Suggestion("country:DK", "Denmark", "country",
|
||||
69 + 56 + 35 + 34 + 16 + 18 + 26 + 10 + 27 + 1)
|
||||
assert tokens("swe")[0] == "country:SE"
|
||||
assert tokens("sverige")[0] == "country:SE"
|
||||
assert tokens("danmark")[0] == "country:DK"
|
||||
|
||||
|
||||
def test_countries_come_first_then_regions_then_towns() -> None:
|
||||
kinds = [s.kind for s in suggest("den", PLACES)]
|
||||
assert kinds == sorted(kinds, key=["country", "region", "place"].index)
|
||||
assert kinds[0] == "country" and "region" in kinds and kinds[-1] == "place"
|
||||
|
||||
|
||||
def test_towns_are_grouped_and_labelled_by_their_commonest_spelling() -> None:
|
||||
lund = [s for s in suggest("lund", PLACES) if s.kind == "place"]
|
||||
assert lund == [Suggestion("place:lund", "Lund", "place", 127 + 29)]
|
||||
kbh = [s for s in suggest("kob", PLACES) if s.kind == "place"]
|
||||
assert kbh[0].label == "København" and kbh[0].token == "place:kobenhavn"
|
||||
|
||||
|
||||
def test_a_postal_district_is_not_a_town_of_its_own() -> None:
|
||||
labels = [s.label for s in suggest("kob", PLACES) if s.kind == "place"]
|
||||
assert "København K" not in labels and "København Ø" not in labels
|
||||
assert [s.label for s in suggest("aarhus", PLACES) if s.kind == "place"] == ["Aarhus"]
|
||||
|
||||
|
||||
def test_spellings_of_one_town_are_one_suggestion() -> None:
|
||||
towns = [s for s in suggest("cop", PLACES) if s.kind == "place"]
|
||||
assert len(towns) == 1
|
||||
# København, K and Ø, the one in a list of three towns; Copenhagen, and "Remote OR Copenhagen".
|
||||
assert towns[0].label == "København" and towns[0].count == 69 + 35 + 34 + 1 + 56 + 1
|
||||
assert [s.token for s in suggest("kobenhavn", PLACES) if s.kind == "place"] \
|
||||
== [towns[0].token]
|
||||
|
||||
|
||||
def test_only_the_first_comma_part_of_each_semicolon_part_is_a_town() -> None:
|
||||
labels = {s.label for q in "abcdefghijklmnopqrstuvwxyz"
|
||||
for s in suggest(q, PLACES, limit=999) if s.kind == "place"}
|
||||
assert {"Aarhus", "Stockholm", "Malmö", "Lund", "Berlin"} <= labels
|
||||
assert not {"Denmark", "Sweden", "Region Central Jutland", "Remote", "Hybrid", "Europe"} \
|
||||
& labels
|
||||
|
||||
|
||||
def test_remote_hybrid_and_country_names_are_no_town() -> None:
|
||||
for q in ("rem", "hyb", "eur", "denm"):
|
||||
assert [s for s in suggest(q, PLACES) if s.kind == "place"] == [], q
|
||||
|
||||
|
||||
def test_a_word_prefix_finds_a_multi_word_town() -> None:
|
||||
assert tokens("haag") == tokens("hague") == tokens("den haag") == tokens("the hag") \
|
||||
== ["place:the hague"]
|
||||
assert tokens("kgs ly") == ["place:kgs lyngby"]
|
||||
assert tokens("lyngby") == ["place:kgs lyngby"]
|
||||
|
||||
|
||||
def test_the_word_must_start_not_end() -> None:
|
||||
assert "place:lund" not in tokens("und")
|
||||
assert "place:billund" not in tokens("lund")
|
||||
assert "place:albertslund" not in tokens("lund")
|
||||
|
||||
|
||||
def test_new_zealand_is_a_country_and_zealand_a_region_only_when_it_has_roles() -> None:
|
||||
assert tokens("zea")[:1] == ["country:NZ"]
|
||||
assert "region:zealand" not in tokens("zea") # no Zealand role in PLACES
|
||||
assert "country:AU" not in tokens("aus") # no role of its own
|
||||
|
||||
|
||||
def test_limit_and_the_empty_query() -> None:
|
||||
assert suggest("", PLACES) == [] and suggest(" ", PLACES) == [] and suggest("---", PLACES) == []
|
||||
assert len(suggest("a", PLACES, limit=3)) == 3
|
||||
assert len(suggest("a", PLACES)) <= 12
|
||||
|
||||
|
||||
def test_towns_are_ordered_by_roles() -> None:
|
||||
towns = [s for s in suggest("l", PLACES) if s.kind == "place"]
|
||||
assert [s.count for s in towns] == sorted((s.count for s in towns), reverse=True)
|
||||
|
||||
|
||||
def test_suggest_takes_any_iterable() -> None:
|
||||
assert suggest("lund", iter(PLACES)) == suggest("lund", PLACES)
|
||||
|
||||
|
||||
def test_a_suggested_token_finds_the_roles_it_counted(table: sqlite3.Connection) -> None:
|
||||
for s in suggest("kob", PLACES) + suggest("lund", PLACES) + suggest("skane", PLACES):
|
||||
assert parse(s.token).kind == s.kind
|
||||
assert found(table, s.token), s.token
|
||||
assert label(s.token) == s.label or s.kind == "place"
|
||||
Reference in new issue
Block a user