Merge main into wp-h-dupes

This commit is contained in:
Wolf committed 2026-09-29 22:07:34 +02:00
commit fdb6264c98
2 files changed
+1851

No files matched your search

+654
View File
@@ -0,0 +1,654 @@
"""Places as a filter: what a board writes in `place`, matched by whole words.
A location line is free text: "København K", "Kgs. Lyngby", "Aarhus N, Region
Central Jutland, Denmark". A substring match finds `Lund` in `Billund` and
`Denmark` in nothing that only names a town. Here a place is folded (casefolded,
marks stripped, `ø` to `o`, `æ` to `ae`, `å` to `a`, punctuation to spaces) and
matched as whole words, so `Lund` is `lund` and no other word. A filter is one
of three tokens, and several tokens are joined by OR:
country:DK the role's country, or a line that names it while `country`
is empty or `EU` (a board that lists several offices)
region:skane any town or name of a region in `REGIONS`
place:malmo the town's fold and its aliases, so `copenhagen`,
`kobenhavn` and `kopenhamn` find each other
`matches` answers whether one role's folded place and country fit a token, in
Python: a region is over a hundred phrases, and as `LIKE` clauses that costs
several hundred milliseconds a query, as a set intersection on the words of a
line it costs a few. `register` gives a connection that answer as the SQL
function `place_match`, and `match_sql` returns the clause that calls it.
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
chip's text back from the address. This module opens no database; the caller
hands over the places it holds, or its connection.
"""
from __future__ import annotations
import re
import sqlite3
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass
from functools import lru_cache
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
"label", "match_sql", "matches", "parse", "register", "suggest"]
def fold(text: str) -> str:
"""`Skåne län` -> `skane lan`, `København K` -> `kobenhavn k`, `Kgs. Lyngby`
-> `kgs lyngby`. What `roles.place_fold` holds. `å` needs no rule of its own:
NFKD splits it into `a` and a ring, and the ring is a mark."""
return _place_fold(text)
# ------------------------------------------------------------------ countries
def _countries() -> dict[str, tuple[str, ...]]:
"""Every code to the names `_COUNTRY_NAMES` gives it, English name first."""
found: dict[str, tuple[str, ...]] = {}
for line in _COUNTRY_NAMES.split("\n"):
if line:
code, rest = line.split(" ", 1)
found[code] = tuple(name.strip() for name in rest.split("|"))
return found
_COUNTRY_LABELS = _countries()
#: A code to the folded names a country goes by.
_COUNTRY_FOLDS: dict[str, tuple[str, ...]] = {
code: tuple(dict.fromkeys(filter(None, map(fold, names))))
for code, names in _COUNTRY_LABELS.items()}
#: What a line has to say for a role of that country whose `country` column is
#: empty or `EU`: a name of the country, or a city `country_of` knows in it. A
#: board that lists offices ("Copenhagen; London; Warsaw") names cities only.
_COUNTRY_MENTIONS: dict[str, tuple[str, ...]] = {
code: tuple(dict.fromkeys((*names, *(fold(c) for c, cc in _CITIES.items() if cc == code))))
for code, names in _COUNTRY_FOLDS.items()}
#: A folded country name to its code.
_COUNTRY_BY_NAME: dict[str, str] = {
name: code for code, names in _COUNTRY_FOLDS.items() for name in names}
#: Words that are not a town: a country, a macro region, how the role is worked.
_NOT_A_TOWN = frozenset(_COUNTRY_BY_NAME) | frozenset(map(fold, _REGIONS)) \
| {"remote", "hybrid"}
# -------------------------------------------------------------------- aliases
#: Towns boards write in more than one language, first spelling shown. Each
#: group is symmetric: any spelling finds the others. The Nordics and the DACH
#: countries in full for their large cities, and the European cities that
#: appear both ways in boards.
_ALIAS_GROUPS: tuple[tuple[str, ...], ...] = (
# Denmark
("Copenhagen", "København", "Kobenhavn", "Köpenhamn", "Kopenhamn", "Kopenhagen",
"Copenhague", "Kopenhaga"),
("Aarhus", "Århus"),
("Aalborg", "Ålborg"),
("Kgs. Lyngby", "Kongens Lyngby", "Lyngby"),
("Taastrup", "Tåstrup"),
("Elsinore", "Helsingør"),
# Sweden
("Gothenburg", "Göteborg", "Goteborg"),
("Stockholm",),
("Malmö", "Malmo"),
("Helsingborg", "Hälsingborg"),
("Västerås", "Vasteras"),
("Linköping", "Linkoping"),
("Örebro", "Orebro"),
("Norrköping", "Norrkoping"),
("Jönköping", "Jonkoping"),
# Norway, Finland, Iceland
("Oslo",),
("Bergen",),
("Trondheim", "Trondhjem"),
("Helsinki", "Helsingfors"),
("Espoo", "Esbo"),
("Vantaa", "Vanda"),
("Reykjavik", "Reykjavík"),
# Germany
("Munich", "München", "Muenchen", "Munchen"),
("Cologne", "Köln", "Koeln", "Koln"),
("Nuremberg", "Nürnberg", "Nuernberg", "Nurnberg"),
("Düsseldorf", "Duesseldorf", "Dusseldorf"),
("Hanover", "Hannover"),
("Mainz", "Mayence"),
("Frankfurt", "Frankfurt am Main"),
("Göttingen", "Goettingen", "Gottingen"),
("Würzburg", "Wuerzburg", "Wurzburg"),
("Tübingen", "Tuebingen", "Tubingen"),
("Lübeck", "Luebeck", "Lubeck"),
("Münster", "Muenster", "Munster"),
# Austria
("Vienna", "Wien", "Vienne"),
("Salzburg",),
("Innsbruck",),
# Switzerland
("Zurich", "Zürich", "Zuerich"),
("Geneva", "Genève", "Geneve", "Genf", "Ginevra"),
("Basel", "Basle", "Bâle"),
("Bern", "Berne"),
("Lucerne", "Luzern"),
("St. Gallen", "Sankt Gallen", "Saint Gall"),
# The rest of Europe
("Prague", "Praha", "Prag"),
("Warsaw", "Warszawa", "Warschau"),
("Krakow", "Kraków", "Cracow"),
("Wroclaw", "Wrocław", "Breslau"),
("Gdansk", "Gdańsk", "Danzig"),
("Poznan", "Poznań", "Posen"),
("Lisbon", "Lisboa", "Lissabon"),
("Milan", "Milano", "Mailand"),
("Rome", "Roma", "Rom"),
("Turin", "Torino"),
("Florence", "Firenze"),
("Naples", "Napoli", "Neapel"),
("Venice", "Venezia", "Venedig"),
("Genoa", "Genova"),
("Brussels", "Bruxelles", "Brussel", "Brüssel"),
("Antwerp", "Antwerpen", "Anvers"),
("Ghent", "Gent", "Gand"),
("Bruges", "Brugge"),
("Liège", "Liege", "Luik", "Lüttich"),
("The Hague", "Den Haag", "s-Gravenhage"),
("Luxembourg", "Luxemburg"),
("Seville", "Sevilla"),
("Athens", "Athina", "Athen"),
("Bucharest", "București", "Bucuresti"),
("Belgrade", "Beograd"),
("Bratislava", "Pressburg"),
("Kyiv", "Kiev"),
("Cluj-Napoca", "Cluj"),
("Timișoara", "Timisoara"),
("Marseille", "Marseilles"),
("Bolzano", "Bozen"),
)
def _variants(text: str) -> tuple[str, ...]:
"""A fold and its `aa` spelling: Danish writes `å` as `aa` (`Aarhus`, `Taastrup`)
and `Århus` and `Tåstrup` fold to `arhus` and `tastrup`."""
return tuple(dict.fromkeys((text, text.replace("aa", "a"))))
def _aliases() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]:
"""Every folded spelling to its whole group (all folded), and to its shown
spelling. The group starts with the first spelling listed."""
groups: dict[str, tuple[str, ...]] = {}
shown: dict[str, str] = {}
for group in _ALIAS_GROUPS:
members = tuple(dict.fromkeys(v for name in group for v in _variants(fold(name))))
for name in group:
shown.setdefault(fold(name), name)
for member in members:
groups[member] = members
return groups, shown
_GROUPS, _SHOWN = _aliases()
def _expand(text: str) -> tuple[str, ...]:
"""A folded town and everything it is also written as."""
return _GROUPS.get(text) or _variants(text)
# -------------------------------------------------------------------- regions
@dataclass(frozen=True)
class Region:
"""A region below a country, or across two. `names` are what a board calls
the region itself, `towns` the places in it; a line is in the region when it
has one of either as whole words, or is in a region it `includes`, by key.
Only `names` are what the region is suggested and read as. Spellings are
shown with their marks and folded when used."""
key: str
label: str
names: tuple[str, ...]
towns: tuple[str, ...]
country: str = ""
includes: tuple[str, ...] = ()
def _split(text: str) -> tuple[str, ...]:
return tuple(part.strip() for part in text.split(",") if part.strip())
_SKANE = Region(
"skane", "Skåne",
_split("Skåne, Skåne län, Skåne County, Region Skåne, Skåne Region, Scania, Skaane"),
_split("""Malmö, Lund, Helsingborg, Kristianstad, Landskrona, Trelleborg, Ängelholm,
Hässleholm, Ystad, Eslöv, Höganäs, Staffanstorp, Lomma, Bjuv, Burlöv, Kävlinge,
Svedala, Vellinge, Simrishamn, Tomelilla, Sjöbo, Svalöv, Klippan, Åstorp,
Bromölla, Osby, Perstorp, Örkelljunga, Östra Göinge, Hörby, Höör, Båstad,
Skurup, Arlöv, Limhamn, Dalby, Åkarp, Bjärred, Höllviken, Ljungbyhed,
Löddeköpinge, Furulund, Genarp, Veberöd, Skanör, Falsterbo, Mölle, Arild"""),
"SE")
_HOVEDSTADEN = Region(
"hovedstaden", "Capital Region of Denmark",
_split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"),
_split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup,
Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte,
Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre,
Høje-Taastrup, Hørsholm, Ishøj, Lyngby-Taarbæk, Rudersdal, Rødovre, Tårnby,
Vallensbæk, Kgs. Lyngby, Kongens Lyngby, Lyngby, Virum, Holte, Nærum, Birkerød,
Vedbæk, Skodsborg, Klampenborg, Charlottenlund, Hellerup, Ordrup, Bagsværd,
Søborg, Mørkhøj, Måløv, Skovlunde, Smørum, Ølstykke, Stenløse, Værløse, Farum,
Slangerup, Hedehusene, Taastrup, Kastrup, Amager, Valby, Vanløse, Brønshøj,
Nørrebro, Østerbro, Vesterbro, Nordhavn, Ørestad, Sydhavn, Islands Brygge,
Refshaleøen, Humlebæk, Kokkedal, Espergærde, Snekkersten, Rungsted, Rungsted Kyst,
Lillerød, Frederiksværk, Hundested, Helsinge, Gilleleje, Græsted, Vallensbæk Strand,
Skævinge, Rønne, Nexø, Allinge, Svaneke, Aakirkeby, Hasle"""),
"DK")
# Region Sjælland, not the island: Copenhagen is on Zealand too, but a board that
# writes "Region Zealand" means the region and the Capital Region has its own key.
# The bare word is left out of `names` since "New Zealand" would match it.
_ZEALAND = Region(
"zealand", "Zealand",
_split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"),
_split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved,
Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg,
Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing,
Haslev, Korsør, Skælskør, Præstø, Stege, Fakse, Rødby, Rødbyhavn, Jyllinge,
Viby Sjælland, Solrød Strand, Fuglebjerg, Gørlev, Dianalund, Jyderup, Hvalsø,
Tølløse, Hørve, Fårevejle, Bjæverskov, Nørre Asmindrup"""),
"DK")
_JUTLAND = Region(
"jutland", "Jutland",
_split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland,
North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland,
East Jutland, West Jutland, Vestjylland, Østjylland"""),
_split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning,
Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov,
Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå,
Hjørring, Frederikshavn, Skagen, Thisted, Struer, Ikast, Brande, Varde, Ribe,
Tønder, Vojens, Hadsten, Hobro, Skanderborg, Odder, Hedensted, Vejen, Ebeltoft,
Lemvig, Rønde, Egå, Lystrup, Hinnerup, Tilst, Hammel, Galten, Sabro, Harlev,
Løsning, Højbjerg, Juelsminde, Nørresundby, Brønderslev, Aars, Nibe, Sæby, Løgstør, Tarm,
Skjern, Padborg, Rødekro, Toftlund, Bramming, Kruså, Brabrand, Ry, Gjern"""),
"DK")
_FUNEN = Region(
"funen", "Funen",
_split("Funen, Fyn, Region Fyn"),
_split("""Odense, Svendborg, Nyborg, Faaborg, Fåborg, Middelfart, Assens, Kerteminde,
Bogense, Langeskov, Ringe, Otterup, Munkebo, Vissenbjerg, Årslev, Tommerup,
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
"DK")
# The Øresund region is the two capital regions either side of the sound. It is
# suggested on its own names only; a line in either region is in it.
_ORESUND = Region(
"oresund", "Øresund",
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"),
(), includes=("hovedstaden", "skane"))
REGIONS: dict[str, Region] = {
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)}
def _phrases(region: Region) -> tuple[str, ...]:
"""Every folded phrase that puts a line in the region, with the other
spellings of its towns, and those of the regions it includes."""
found: dict[str, None] = {}
for text in (*region.names, *region.towns):
for form in _expand(fold(text)):
found[form] = None
for key in region.includes:
found.update(dict.fromkeys(_phrases(REGIONS[key])))
return tuple(found)
_PHRASES: dict[str, tuple[str, ...]] = {key: _phrases(r) for key, r in REGIONS.items()}
#: A folded key, label or name to the region key; a name in two regions ("Capital
#: Region of Denmark" is also in Øresund) belongs to the one that lists it first.
_REGION_BY_NAME: dict[str, str] = {}
for _key, _region in REGIONS.items():
for _name in (_region.key, _region.label, *_region.names):
_REGION_BY_NAME.setdefault(fold(_name), _key)
#: What a place is shown as when a chip is read from the address: every alias
#: spelling, every region and every town of one, as written above.
_TOWN_LABELS: dict[str, str] = dict(_SHOWN)
for _region in REGIONS.values():
for _town in (_region.label, *_region.towns, *_region.names):
_TOWN_LABELS.setdefault(fold(_town), _town)
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
def _group(folded: str) -> str:
"""The key of the town a fold is: the first spelling of its alias group, else
the fold with `aa` as `a`."""
return _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
def _known_towns() -> dict[str, str]:
"""Every town the tables name, as a fold, to its group: the alias groups, the
towns of the regions and the cities `country_of` knows."""
names = [*_GROUPS, *(fold(city) for city in _CITIES),
*(fold(town) for region in REGIONS.values() for town in region.towns)]
return {form: _group(form) for name in names for form in _variants(name)}
_KNOWN = _known_towns()
_KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN)
# --------------------------------------------------------------------- tokens
@dataclass(frozen=True)
class Token:
"""One place filter. `value` is a country code, a region key or a fold."""
kind: str
value: str
label: str
def __str__(self) -> str:
return f"{self.kind}:{self.value}"
def _region_key(text: str) -> str:
folded = fold(text)
return _REGION_BY_NAME.get(folded, folded)
def parse(token: str) -> Token:
"""`country:DK`, `region:skane` or `place:malmo`. Bare text, or a prefix that
is none of these, is a place: `Skåne` is `place:skane`. A country by name
(`country:Denmark`) and a region by label (`region:Skåne`) are read as well."""
kind, sep, value = (token or "").partition(":")
kind = kind.strip().lower()
if not sep or kind not in ("country", "region", "place"):
kind, value = "place", token or ""
value = value.strip()
if kind == "country":
code = value.upper()
if code not in _COUNTRY_LABELS:
code = _COUNTRY_BY_NAME.get(fold(value), code)
return Token("country", code, _COUNTRY_LABELS.get(code, (code,))[0])
if kind == "region":
key = _region_key(value)
return Token("region", key, REGIONS[key].label if key in REGIONS else key)
folded = fold(value)
return Token("place", folded, _TOWN_LABELS.get(folded) or folded.title())
def label(token: str | Token) -> str:
"""The text of a chip read from the address: `country:DK` is `Denmark`,
`region:skane` is `Skåne`, `place:malmo` is `Malmö`. A town the tables do not
know is its fold in title case."""
return (token if isinstance(token, Token) else parse(token)).label
# ------------------------------------------------------------------- matching
SQL_FUNCTION = "place_match"
def known(token: str) -> bool:
"""Whether the token names something: a country of the table (or `EU`), a
region of `REGIONS`, a place with a value. `region:atlantis`, `country:ZZ`
and `place:` are not."""
return _known(parse(token))
def _known(tok: Token) -> bool:
if tok.kind == "country":
return tok.value in _COUNTRY_LABELS or tok.value == "EU"
if tok.kind == "region":
return tok.value in REGIONS
return bool(tok.value)
@dataclass(frozen=True)
class _Matcher:
"""What a token asks of a role. `phrases` are whole-word phrases of the
folded place; `countries` are `(code, mentions)`: the role's own country, or
a line with one of the mentions while `country` is empty or `EU`, since a
board listing several offices has no single country."""
phrases: frozenset[str]
countries: tuple[tuple[str, frozenset[str]], ...]
longest: int
@lru_cache(maxsize=1024)
def _matcher(token: str) -> _Matcher | None:
"""The token read once: `None` when it names nothing.
A `place:` value that is also a region or a country is matched as that too:
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
written as a bare word means what the owner meant by it."""
tok = parse(token)
if not _known(tok):
return None
phrases: set[str] = set()
countries: list[tuple[str, frozenset[str]]] = []
if tok.kind == "country":
countries.append((tok.value, frozenset(_COUNTRY_MENTIONS.get(tok.value, ()))))
elif tok.kind == "region":
phrases.update(_PHRASES[tok.value])
else:
phrases.update(_expand(tok.value))
phrases.update(_PHRASES.get(_REGION_BY_NAME.get(tok.value, ""), ()))
if code := _COUNTRY_BY_NAME.get(tok.value):
countries.append((code, frozenset(_COUNTRY_MENTIONS.get(code, ()))))
phrases.discard("")
longest = max((len(p.split()) for p in (*phrases, *(m for _, ms in countries for m in ms))),
default=1)
return _Matcher(frozenset(phrases), tuple(countries), longest)
@lru_cache(maxsize=32768)
def _runs(place_fold: str, longest: int) -> frozenset[str]:
"""Every run of up to `longest` words of a folded line, each a whole-word
phrase the line holds."""
words = place_fold.split()
return frozenset(" ".join(words[i:i + n])
for n in range(1, longest + 1) for i in range(len(words) - n + 1))
def matches(place_fold: str, country: str, token: str) -> bool:
"""Whether a role whose folded place and country are these is found by the
token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or
a Danish city) while `country` is empty or `EU`. `region:skane` is any of
its names or towns as whole words, `place:malmo` the town and its aliases.
A token that names nothing (`known` is false) matches no role."""
match = _matcher(token)
if match is None:
return False
runs = None
if match.phrases:
runs = _runs(place_fold, match.longest)
if not match.phrases.isdisjoint(runs):
return True
for code, mentions in match.countries:
if country == code:
return True
if mentions and country in ("", "EU"):
runs = _runs(place_fold, match.longest) if runs is None else runs
if not mentions.isdisjoint(runs):
return True
return False
def _place_match(place_fold: str | None, country: str | None, token: str) -> int:
return int(matches(place_fold or "", country or "", token))
def register(conn: sqlite3.Connection) -> None:
"""Give a connection `place_match(place_fold, country, token)`, which is 1 when
`matches` is true. `match_sql` returns a clause that calls it."""
conn.create_function(SQL_FUNCTION, 3, _place_match, deterministic=True)
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
-> tuple[str, list]:
"""SQL over the two columns for one token, with one `?` parameter. It calls
`place_match`, so the connection needs `register`. Several tokens are joined
with OR by the caller. Bare text is a place. A token that names nothing
(`known` is false) is `0`, which matches no role."""
tok = parse(token)
if not _known(tok):
return "0", []
return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)]
# ---------------------------------------------------------------- suggestions
@dataclass(frozen=True)
class Suggestion:
"""`token` is what the chip holds, `kind` is `country`, `region` or `place`,
`count` the open roles it stands for."""
token: str
label: str
kind: str
count: int
def _prefixes(q: str, candidates: Iterable[str]) -> bool:
"""`q` starts a word of a candidate, or a run of words: `kgs ly` finds
`kgs lyngby`, `hague` finds `the hague`."""
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
#: Where one town ends in a line: `;` and `|` between places, and `or` / `eller` /
#: `og` between two towns in one ("København V eller Aalborg").
_PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Danish boards append "and the possibility of working from home".
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
#: A country or "Remote" ahead of the town, "Sweden - Lund", "Remote-New Zealand".
_LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$")
def _towns(place: str) -> list[str]:
"""The first comma part of each place of a line: `Aarhus N, Region Central
Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal
district is dropped, since `place:kobenhavn` finds `København K` anyway."""
found = []
for part in _PART.split(_AND_HOME.sub("", place or "")):
head = part.split(",")[0].strip()
if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN:
head = lead[2]
head = _DISTRICT.sub("", head)
if fold(head) and fold(head) not in _NOT_A_TOWN:
found.append(head)
return found
def _towns_in(folded: str) -> list[str]:
"""The known towns a fold holds as whole words, longest first from the left,
as their groups: `munich berlin` is Munich and Berlin, `copenhagen office` is
Copenhagen, `hoje taastrup` is one town and not also Taastrup."""
words = folded.split()
found: list[str] = []
i = 0
while i < len(words):
for n in range(min(_KNOWN_LONGEST, len(words) - i), 0, -1):
if (group := _KNOWN.get(" ".join(words[i:i + n]))) is not None:
found.append(group)
i += n
break
else:
i += 1
return list(dict.fromkeys(found))
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
"""Open roles per region, over folded lines. A line is cut into every run of
words once, and each region asks whether it holds one of its phrases."""
sets = {key: frozenset(_PHRASES[key]) for key in keys}
longest = max((len(p.split()) for phrases in sets.values() for p in phrases), default=1)
counts: Counter[str] = Counter()
for text, roles in places:
runs = _runs(text, longest)
for key, phrases in sets.items():
if not phrases.isdisjoint(runs):
counts[key] += roles
return counts
def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
-> list[Suggestion]:
"""Tokens for the text typed so far. `places` is `(place, country, open
roles)`. Countries first, then regions, then towns, each by open roles.
A country is suggested when `q` starts a word of one of its names, with the
roles that carry its code. A region likewise on its label and names, with the
roles whose line is in it. A town is the first comma part of each `;` part of
a line, grouped by fold and by alias (`Copenhagen` and `København` are one
town, shown as its commonest spelling). Countries, regions and towns with no
open role are left out: a chip that finds nothing helps nobody."""
text = fold(q)
if not text:
return []
rows = [(place or "", country or "", int(roles)) for place, country, roles in places]
by_country: Counter[str] = Counter()
for _, country, roles in rows:
by_country[country] += roles
countries = [Suggestion(f"country:{code}", names[0], "country", by_country[code])
for code, names in _COUNTRY_LABELS.items()
if by_country[code] and _prefixes(text, _COUNTRY_FOLDS[code])]
named = [key for key, r in REGIONS.items()
if _prefixes(text, [fold(r.label), *map(fold, r.names)])]
counts = _region_counts(named, [(fold(place), roles) for place, _, roles in rows])
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
for key in named if counts[key]]
# A line's town is a group. A line that is a town is that group's spelling;
# one that only holds a known town ("Copenhagen Office", "Munich - Berlin")
# counts under each such town and shows as none; anything else is its own.
counts_by_town: Counter[str] = Counter()
spellings: dict[str, Counter[str]] = {}
written: dict[str, set[str]] = {}
for place, _, roles in rows:
for town in _towns(place):
folded = fold(town)
if folded not in _KNOWN and (inside := _towns_in(folded)):
for group in inside:
counts_by_town[group] += roles
continue
group = _group(folded)
counts_by_town[group] += roles
spellings.setdefault(group, Counter())[town] += roles
written.setdefault(group, set()).add(folded)
towns = []
for group, total in counts_by_town.items():
forms = {group, *written.get(group, ())}
if not _prefixes(text, [form for folded in forms for form in _expand(folded)]):
continue
seen = spellings.get(group)
shown = min(seen, key=lambda s: (-seen[s], s)) if seen \
else _TOWN_LABELS.get(group) or group.title()
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total))
countries.sort(key=lambda s: (-s.count, s.label))
regions.sort(key=lambda s: (-s.count, s.label))
towns.sort(key=lambda s: (-s.count, s.label))
return (countries + regions + towns)[:limit]
+1197
View File
File diff suppressed because it is too large. Load diff