From ebb1f560de543700d278caf490464abab1daa8da Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 21:58:32 +0200 Subject: [PATCH 1/2] Add places: folded whole-word place matching, country, region and town tokens, suggestions Co-Authored-By: Claude --- app/places.py | 537 +++++++++++++++++++++++++++++++++++++++++++ tests/test_places.py | 453 ++++++++++++++++++++++++++++++++++++ 2 files changed, 990 insertions(+) create mode 100644 app/places.py create mode 100644 tests/test_places.py diff --git a/app/places.py b/app/places.py new file mode 100644 index 0000000..ae564a6 --- /dev/null +++ b/app/places.py @@ -0,0 +1,537 @@ +"""Places as a filter: what a board writes in `place`, matched by whole words. + +A location line is free text: "København K", "Kgs. Lyngby", "Aarhus N, Region +Central Jutland, Denmark". A substring match finds `Lund` in `Billund` and +`Denmark` in nothing that only names a town. Here a place is folded (casefolded, +marks stripped, `ø` to `o`, `æ` to `ae`, `å` to `a`, punctuation to spaces) and +matched as whole words, so `Lund` is `lund` and no other word. A filter is one +of three tokens, and several tokens are joined by OR: + + country:DK the role's country, or a line that names it while `country` + is empty or `EU` (a board that lists several offices) + region:skane any town or name of a region in `REGIONS` + place:malmo the town's fold and its aliases, so `copenhagen`, + `kobenhavn` and `kopenhamn` find each other + +`match_sql` turns a token into SQL over `roles.place_fold` and `roles.country`; +`suggest` turns typed text into tokens for the autocomplete; `label` reads a +chip's text back from the address. No database access here, the caller hands +over the places it holds. +""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Iterable +from dataclasses import dataclass + +from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold + +__all__ = ["REGIONS", "Region", "Suggestion", "Token", "fold", "label", "match_sql", + "parse", "suggest"] + + +def fold(text: str) -> str: + """`Skåne län` -> `skane lan`, `København K` -> `kobenhavn k`, `Kgs. Lyngby` + -> `kgs lyngby`. What `roles.place_fold` holds. `å` needs no rule of its own: + NFKD splits it into `a` and a ring, and the ring is a mark.""" + return _place_fold(text) + + +# ------------------------------------------------------------------ countries + + +def _countries() -> dict[str, tuple[str, ...]]: + """Every code to the names `_COUNTRY_NAMES` gives it, English name first.""" + found: dict[str, tuple[str, ...]] = {} + for line in _COUNTRY_NAMES.split("\n"): + if line: + code, rest = line.split(" ", 1) + found[code] = tuple(name.strip() for name in rest.split("|")) + return found + + +_COUNTRY_LABELS = _countries() + +#: A code to the folded names a country goes by. +_COUNTRY_FOLDS: dict[str, tuple[str, ...]] = { + code: tuple(dict.fromkeys(filter(None, map(fold, names)))) + for code, names in _COUNTRY_LABELS.items()} + +#: What a line has to say for a role of that country whose `country` column is +#: empty or `EU`: a name of the country, or a city `country_of` knows in it. A +#: board that lists offices ("Copenhagen; London; Warsaw") names cities only. +_COUNTRY_MENTIONS: dict[str, tuple[str, ...]] = { + code: tuple(dict.fromkeys((*names, *(fold(c) for c, cc in _CITIES.items() if cc == code)))) + for code, names in _COUNTRY_FOLDS.items()} + +#: A folded country name to its code. +_COUNTRY_BY_NAME: dict[str, str] = { + name: code for code, names in _COUNTRY_FOLDS.items() for name in names} + +#: Words that are not a town: a country, a macro region, how the role is worked. +_NOT_A_TOWN = frozenset(_COUNTRY_BY_NAME) | frozenset(map(fold, _REGIONS)) \ + | {"remote", "hybrid"} + + +# -------------------------------------------------------------------- aliases + +#: Towns boards write in more than one language, first spelling shown. Each +#: group is symmetric: any spelling finds the others. The Nordics and the DACH +#: countries in full for their large cities, and the European cities that +#: appear both ways in boards. +_ALIAS_GROUPS: tuple[tuple[str, ...], ...] = ( + # Denmark + ("Copenhagen", "København", "Kobenhavn", "Köpenhamn", "Kopenhamn", "Kopenhagen", + "Copenhague", "Kopenhaga"), + ("Aarhus", "Århus"), + ("Aalborg", "Ålborg"), + ("Kgs. Lyngby", "Kongens Lyngby", "Lyngby"), + ("Taastrup", "Tåstrup"), + ("Elsinore", "Helsingør"), + # Sweden + ("Gothenburg", "Göteborg", "Goteborg"), + ("Stockholm",), + ("Malmö", "Malmo"), + ("Helsingborg", "Hälsingborg"), + ("Västerås", "Vasteras"), + ("Linköping", "Linkoping"), + ("Örebro", "Orebro"), + ("Norrköping", "Norrkoping"), + ("Jönköping", "Jonkoping"), + # Norway, Finland, Iceland + ("Oslo",), + ("Bergen",), + ("Trondheim", "Trondhjem"), + ("Helsinki", "Helsingfors"), + ("Espoo", "Esbo"), + ("Vantaa", "Vanda"), + ("Reykjavik", "Reykjavík"), + # Germany + ("Munich", "München", "Muenchen", "Munchen"), + ("Cologne", "Köln", "Koeln", "Koln"), + ("Nuremberg", "Nürnberg", "Nuernberg", "Nurnberg"), + ("Düsseldorf", "Duesseldorf", "Dusseldorf"), + ("Hanover", "Hannover"), + ("Mainz", "Mayence"), + ("Frankfurt", "Frankfurt am Main"), + ("Göttingen", "Goettingen", "Gottingen"), + ("Würzburg", "Wuerzburg", "Wurzburg"), + ("Tübingen", "Tuebingen", "Tubingen"), + ("Lübeck", "Luebeck", "Lubeck"), + ("Münster", "Muenster", "Munster"), + # Austria + ("Vienna", "Wien", "Vienne"), + ("Salzburg",), + ("Innsbruck",), + # Switzerland + ("Zurich", "Zürich", "Zuerich"), + ("Geneva", "Genève", "Geneve", "Genf", "Ginevra"), + ("Basel", "Basle", "Bâle"), + ("Bern", "Berne"), + ("Lucerne", "Luzern"), + ("St. Gallen", "Sankt Gallen", "Saint Gall"), + # The rest of Europe + ("Prague", "Praha", "Prag"), + ("Warsaw", "Warszawa", "Warschau"), + ("Krakow", "Kraków", "Cracow"), + ("Wroclaw", "Wrocław", "Breslau"), + ("Gdansk", "Gdańsk", "Danzig"), + ("Poznan", "Poznań", "Posen"), + ("Lisbon", "Lisboa", "Lissabon"), + ("Milan", "Milano", "Mailand"), + ("Rome", "Roma", "Rom"), + ("Turin", "Torino"), + ("Florence", "Firenze"), + ("Naples", "Napoli", "Neapel"), + ("Venice", "Venezia", "Venedig"), + ("Genoa", "Genova"), + ("Brussels", "Bruxelles", "Brussel", "Brüssel"), + ("Antwerp", "Antwerpen", "Anvers"), + ("Ghent", "Gent", "Gand"), + ("Bruges", "Brugge"), + ("Liège", "Liege", "Luik", "Lüttich"), + ("The Hague", "Den Haag", "s-Gravenhage"), + ("Luxembourg", "Luxemburg"), + ("Seville", "Sevilla"), + ("Athens", "Athina", "Athen"), + ("Bucharest", "București", "Bucuresti"), + ("Belgrade", "Beograd"), + ("Bratislava", "Pressburg"), + ("Kyiv", "Kiev"), + ("Cluj-Napoca", "Cluj"), + ("Timișoara", "Timisoara"), + ("Marseille", "Marseilles"), + ("Bolzano", "Bozen"), +) + + +def _variants(text: str) -> tuple[str, ...]: + """A fold and its `aa` spelling: Danish writes `å` as `aa` (`Aarhus`, `Taastrup`) + and `Århus` and `Tåstrup` fold to `arhus` and `tastrup`.""" + return tuple(dict.fromkeys((text, text.replace("aa", "a")))) + + +def _aliases() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]: + """Every folded spelling to its whole group (all folded), and to its shown + spelling. The group starts with the first spelling listed.""" + groups: dict[str, tuple[str, ...]] = {} + shown: dict[str, str] = {} + for group in _ALIAS_GROUPS: + members = tuple(dict.fromkeys(v for name in group for v in _variants(fold(name)))) + for name in group: + shown.setdefault(fold(name), name) + for member in members: + groups[member] = members + return groups, shown + + +_GROUPS, _SHOWN = _aliases() + + +def _expand(text: str) -> tuple[str, ...]: + """A folded town and everything it is also written as.""" + return _GROUPS.get(text) or _variants(text) + + +# -------------------------------------------------------------------- regions + + +@dataclass(frozen=True) +class Region: + """A region below a country, or across two. `names` are what a board calls + the region itself, `towns` the places in it; a line is in the region when it + has one of either as whole words. Spellings are shown with their marks and + folded when used.""" + + key: str + label: str + names: tuple[str, ...] + towns: tuple[str, ...] + country: str = "" + + +def _split(text: str) -> tuple[str, ...]: + return tuple(part.strip() for part in text.split(",") if part.strip()) + + +_SKANE = Region( + "skane", "Skåne", + _split("Skåne, Skåne län, Skåne County, Region Skåne, Skåne Region, Scania, Skaane"), + _split("""Malmö, Lund, Helsingborg, Kristianstad, Landskrona, Trelleborg, Ängelholm, + Hässleholm, Ystad, Eslöv, Höganäs, Staffanstorp, Lomma, Bjuv, Burlöv, Kävlinge, + Svedala, Vellinge, Simrishamn, Tomelilla, Sjöbo, Svalöv, Klippan, Åstorp, + Bromölla, Osby, Perstorp, Örkelljunga, Östra Göinge, Hörby, Höör, Båstad, + Skurup, Arlöv, Limhamn, Dalby, Åkarp, Bjärred, Höllviken, Ljungbyhed, + Löddeköpinge, Furulund, Genarp, Veberöd, Skanör, Falsterbo, Mölle, Arild"""), + "SE") + +_HOVEDSTADEN = Region( + "hovedstaden", "Capital Region of Denmark", + _split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"), + _split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup, + Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte, + Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre, + Høje-Taastrup, Hørsholm, Ishøj, Lyngby-Taarbæk, Rudersdal, Rødovre, Tårnby, + Vallensbæk, Kgs. Lyngby, Kongens Lyngby, Lyngby, Virum, Holte, Nærum, Birkerød, + Vedbæk, Skodsborg, Klampenborg, Charlottenlund, Hellerup, Ordrup, Bagsværd, + Søborg, Mørkhøj, Måløv, Skovlunde, Smørum, Ølstykke, Stenløse, Værløse, Farum, + Slangerup, Hedehusene, Taastrup, Kastrup, Amager, Valby, Vanløse, Brønshøj, + Nørrebro, Østerbro, Vesterbro, Nordhavn, Ørestad, Sydhavn, Islands Brygge, + Refshaleøen, Humlebæk, Kokkedal, Espergærde, Snekkersten, Rungsted, Rungsted Kyst, + Lillerød, Frederiksværk, Hundested, Helsinge, Gilleleje, Græsted, Vallensbæk Strand, + Skævinge, Rønne, Nexø, Allinge, Svaneke, Aakirkeby, Hasle"""), + "DK") + +# Region Sjælland, not the island: Copenhagen is on Zealand too, but a board that +# writes "Region Zealand" means the region and the Capital Region has its own key. +# The bare word is left out of `names` since "New Zealand" would match it. +_ZEALAND = Region( + "zealand", "Zealand", + _split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"), + _split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved, + Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg, + Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing, + Haslev, Korsør, Skælskør, Præstø, Stege, Fakse, Rødby, Rødbyhavn, Jyllinge, + Viby Sjælland, Solrød Strand, Fuglebjerg, Gørlev, Dianalund, Jyderup, Hvalsø, + Tølløse, Hørve, Fårevejle, Bjæverskov, Nørre Asmindrup"""), + "DK") + +_JUTLAND = Region( + "jutland", "Jutland", + _split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland, + North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland, + East Jutland, West Jutland, Vestjylland, Østjylland"""), + _split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning, + Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov, + Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå, + Hjørring, Frederikshavn, Skagen, Thisted, Struer, Ikast, Brande, Varde, Ribe, + Tønder, Vojens, Hadsten, Hobro, Skanderborg, Odder, Hedensted, Vejen, Ebeltoft, + Lemvig, Rønde, Egå, Lystrup, Hinnerup, Tilst, Hammel, Galten, Sabro, Harlev, + Løsning, Højbjerg, Juelsminde, Nørresundby, Brønderslev, Aars, Nibe, Sæby, Løgstør, Tarm, + Skjern, Padborg, Rødekro, Toftlund, Bramming, Kruså, Brabrand, Ry, Gjern"""), + "DK") + +_FUNEN = Region( + "funen", "Funen", + _split("Funen, Fyn, Region Fyn"), + _split("""Odense, Svendborg, Nyborg, Faaborg, Fåborg, Middelfart, Assens, Kerteminde, + Bogense, Langeskov, Ringe, Otterup, Munkebo, Vissenbjerg, Årslev, Tommerup, + Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""), + "DK") + +# The Øresund region is the two capital regions either side of the sound. +_ORESUND = Region( + "oresund", "Øresund", + _split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen") + + _HOVEDSTADEN.names + _SKANE.names, + _HOVEDSTADEN.towns + _SKANE.towns) + +REGIONS: dict[str, Region] = { + r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)} + + +def _phrases(region: Region) -> tuple[str, ...]: + """Every folded phrase that puts a line in the region, with the other + spellings of its towns.""" + found: dict[str, None] = {} + for text in (*region.names, *region.towns): + for form in _expand(fold(text)): + found[form] = None + return tuple(found) + + +_PHRASES: dict[str, tuple[str, ...]] = {key: _phrases(r) for key, r in REGIONS.items()} + +#: A folded key, label or name to the region key; a name in two regions ("Capital +#: Region of Denmark" is also in Øresund) belongs to the one that lists it first. +_REGION_BY_NAME: dict[str, str] = {} +for _key, _region in REGIONS.items(): + for _name in (_region.key, _region.label, *_region.names): + _REGION_BY_NAME.setdefault(fold(_name), _key) + +#: What a place is shown as when a chip is read from the address: every alias +#: spelling, every region and every town of one, as written above. +_TOWN_LABELS: dict[str, str] = dict(_SHOWN) +for _region in REGIONS.values(): + for _town in (_region.label, *_region.towns, *_region.names): + _TOWN_LABELS.setdefault(fold(_town), _town) + +_NOT_A_TOWN |= frozenset(_REGION_BY_NAME) + + +# --------------------------------------------------------------------- tokens + + +@dataclass(frozen=True) +class Token: + """One place filter. `value` is a country code, a region key or a fold.""" + + kind: str + value: str + label: str + + def __str__(self) -> str: + return f"{self.kind}:{self.value}" + + +def _region_key(text: str) -> str: + folded = fold(text) + return _REGION_BY_NAME.get(folded, folded) + + +def parse(token: str) -> Token: + """`country:DK`, `region:skane` or `place:malmo`. Bare text, or a prefix that + is none of these, is a place: `Skåne` is `place:skane`. A country by name + (`country:Denmark`) and a region by label (`region:Skåne`) are read as well.""" + kind, sep, value = (token or "").partition(":") + kind = kind.strip().lower() + if not sep or kind not in ("country", "region", "place"): + kind, value = "place", token or "" + value = value.strip() + if kind == "country": + code = value.upper() + if code not in _COUNTRY_LABELS: + code = _COUNTRY_BY_NAME.get(fold(value), code) + return Token("country", code, _COUNTRY_LABELS.get(code, (code,))[0]) + if kind == "region": + key = _region_key(value) + return Token("region", key, REGIONS[key].label if key in REGIONS else key) + folded = fold(value) + return Token("place", folded, _TOWN_LABELS.get(folded) or folded.title()) + + +def label(token: str | Token) -> str: + """The text of a chip read from the address: `country:DK` is `Denmark`, + `region:skane` is `Skåne`, `place:malmo` is `Malmö`. A town the tables do not + know is its fold in title case.""" + return (token if isinstance(token, Token) else parse(token)).label + + +# ------------------------------------------------------------------------ SQL + +_NEVER = ("0", []) + + +def _escape(word: str) -> str: + """A folded word holds no wildcard, but a caller may hand over raw text.""" + return re.sub(r"([\\%_])", r"\\\1", word) + + +def _words(column: str, phrases: Iterable[str]) -> tuple[str, list]: + """Any of the phrases as whole words of the column. A multi-word phrase is + a phrase: `kgs lyngby` needs both words, side by side.""" + phrases = [p for p in dict.fromkeys(phrases) if p] + if not phrases: + return _NEVER + clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'" + return " OR ".join([clause] * len(phrases)), [f"% {_escape(p)} %" for p in phrases] + + +def _country(code: str, place: str, country: str) -> tuple[str, list]: + """The role's own country, or a line that names the country while `country` + is empty or `EU`: a board listing several offices has no single country.""" + names, params = _words(place, _COUNTRY_MENTIONS.get(code, ())) + if not params: + return f"{country} = ?", [code] + return f"{country} = ? OR ({country} IN ('', 'EU') AND ({names}))", [code, *params] + + +def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \ + -> tuple[str, list]: + """SQL over the two columns for one token, parenthesised, with `?` parameters. + Several tokens are joined with OR by the caller. Bare text is a place. + + A `place:` value that is also a region or a country is matched as that too: + `place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate + written as a bare word means what the owner meant by it. A token that names + nothing known (`region:atlantis`, an empty value) matches no role.""" + tok = parse(token) + if tok.kind == "country": + sql, params = _country(tok.value, place, country) + elif tok.kind == "region": + sql, params = _words(place, _PHRASES.get(tok.value, ())) + else: + region = _REGION_BY_NAME.get(tok.value) + sql, params = _words(place, [*_expand(tok.value), *_PHRASES.get(region, ())]) \ + if tok.value else _NEVER + if tok.value in _COUNTRY_BY_NAME: + more, extra = _country(_COUNTRY_BY_NAME[tok.value], place, country) + sql, params = f"{sql} OR {more}", [*params, *extra] + return f"({sql})", params + + +# ---------------------------------------------------------------- suggestions + + +@dataclass(frozen=True) +class Suggestion: + """`token` is what the chip holds, `kind` is `country`, `region` or `place`, + `count` the open roles it stands for.""" + + token: str + label: str + kind: str + count: int + + +def _prefixes(q: str, candidates: Iterable[str]) -> bool: + """`q` starts a word of a candidate, or a run of words: `kgs ly` finds + `kgs lyngby`, `hague` finds `the hague`.""" + return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates) + + +#: Where one town ends in a line: `;` between places, and `or` / `eller` / `og` +#: between two towns in one ("København V eller Aalborg"). +_PART = re.compile(r"\s*;\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE) + +#: Danish boards append "and the possibility of working from home". +_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE) + +#: A postal district after the town, "København K", "Aarhus C", "Odense SV". +_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$") + +#: A country or "Remote" ahead of the town, "Sweden - Lund", "Remote-New Zealand". +_LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$") + + +def _towns(place: str) -> list[str]: + """The first comma part of each place of a line: `Aarhus N, Region Central + Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal + district is dropped, since `place:kobenhavn` finds `København K` anyway.""" + found = [] + for part in _PART.split(_AND_HOME.sub("", place or "")): + head = part.split(",")[0].strip() + if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN: + head = lead[2] + head = _DISTRICT.sub("", head) + if fold(head) and fold(head) not in _NOT_A_TOWN: + found.append(head) + return found + + +def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]: + """Open roles per region, over folded lines. A line is cut into every run of + words once, and each region asks whether it holds one of its phrases.""" + sets = {key: set(_PHRASES[key]) for key in keys} + longest = max((len(p.split()) for key in sets for p in sets[key]), default=1) + counts: Counter[str] = Counter() + for text, roles in places: + words = text.split() + runs = {" ".join(words[i:i + n]) + for n in range(1, longest + 1) for i in range(len(words) - n + 1)} + for key, phrases in sets.items(): + if not phrases.isdisjoint(runs): + counts[key] += roles + return counts + + +def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \ + -> list[Suggestion]: + """Tokens for the text typed so far. `places` is `(place, country, open + roles)`. Countries first, then regions, then towns, each by open roles. + + A country is suggested when `q` starts a word of one of its names, with the + roles that carry its code. A region likewise on its label and names, with the + roles whose line is in it. A town is the first comma part of each `;` part of + a line, grouped by fold and by alias (`Copenhagen` and `København` are one + town, shown as its commonest spelling). Countries, regions and towns with no + open role are left out: a chip that finds nothing helps nobody.""" + text = fold(q) + if not text: + return [] + rows = [(place or "", country or "", int(roles)) for place, country, roles in places] + + by_country: Counter[str] = Counter() + for _, country, roles in rows: + by_country[country] += roles + countries = [Suggestion(f"country:{code}", names[0], "country", by_country[code]) + for code, names in _COUNTRY_LABELS.items() + if by_country[code] and _prefixes(text, _COUNTRY_FOLDS[code])] + + named = [key for key, r in REGIONS.items() + if _prefixes(text, [fold(r.label), *map(fold, r.names)])] + counts = _region_counts(named, [(fold(place), roles) for place, _, roles in rows]) + regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key]) + for key in named if counts[key]] + + spellings: dict[str, Counter[str]] = {} + written: dict[str, set[str]] = {} + for place, _, roles in rows: + for town in _towns(place): + folded = fold(town) + group = _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1] + spellings.setdefault(group, Counter())[town] += roles + written.setdefault(group, set()).add(folded) + towns = [] + for group, seen in spellings.items(): + if not _prefixes(text, [form for folded in written[group] for form in _expand(folded)]): + continue + shown = min(seen, key=lambda s: (-seen[s], s)) + towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", sum(seen.values()))) + + countries.sort(key=lambda s: (-s.count, s.label)) + regions.sort(key=lambda s: (-s.count, s.label)) + towns.sort(key=lambda s: (-s.count, s.label)) + return (countries + regions + towns)[:limit] diff --git a/tests/test_places.py b/tests/test_places.py new file mode 100644 index 0000000..d1f83df --- /dev/null +++ b/tests/test_places.py @@ -0,0 +1,453 @@ +"""`app.places`: folding, the three token kinds run as SQL, and the suggestions. + +The rows are `place` strings as boards write them, taken from the live database. +Each token's SQL runs against an in-memory table shaped like `roles`.""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from app import places +from app.places import REGIONS, Suggestion, Token, fold, label, match_sql, parse, suggest + +#: (place, country) as a board wrote them and `country_of` read them. +ROWS = [ + ("København", "DK"), ("Copenhagen, Denmark", "DK"), ("Humlebæk, Denmark", "DK"), + ("København K", "DK"), ("København Ø", "DK"), ("Ballerup", "DK"), ("Denmark", "DK"), + ("Aarhus N, Region Central Jutland, Denmark", "DK"), ("Aarhus C, Denmark", "DK"), + ("Billund, Denmark", "DK"), ("Kgs. Lyngby", "DK"), ("Kongens Lyngby", "DK"), + ("Måløv, Capital Region of Denmark, Denmark", "DK"), ("Søborg", "DK"), + ("Kalundborg, Region Zealand, Denmark", "DK"), ("Albertslund", "DK"), + ("Odense, REGION_DNK 1852 South Denmark, Denmark", "DK"), ("Nakskov", "DK"), + ("Nordhavn og mulighed for hjemmearbejde", "DK"), ("Remote OR Copenhagen", "DK"), + ("Malmö", "SE"), ("Lund", "SE"), ("Sweden - Lund", "SE"), ("Helsingborg", "SE"), + ("Malmö, Skåne County, Sweden", "SE"), ("Skåne län, Sweden", "SE"), + ("Höganäs, Sweden", "SE"), ("Stockholm, Sweden", "SE"), ("Gothenburg, Sweden", "SE"), + ("Aarhus, Denmark; København, Denmark; Stockholm, Sweden", "EU"), + ("Copenhagen; London; Warsaw; Milano", "EU"), + ("Copenhagen; Warsaw; New York", ""), + ("Berlin, Germany; Munich, Germany", "DE"), ("Berlin, Germany", "DE"), + ("München", "DE"), ("Munich, Bavaria, Germany", "DE"), ("Köln", "DE"), ("Cologne", "DE"), + ("Zürich", "CH"), ("Zurich, Switzerland", "CH"), ("Genève", "CH"), ("Geneva", "CH"), + ("Wien", "AT"), ("Vienna, Austria", "AT"), + ("Auckland, New Zealand", "NZ"), ("Remote", ""), ("Hybrid", ""), ("Europe", ""), + ("Remote - Denmark", "DK"), ("", ""), +] + + +@pytest.fixture() +def table() -> sqlite3.Connection: + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") + db.executemany("INSERT INTO roles VALUES (?, ?, ?)", + [(place, fold(place), country) for place, country in ROWS]) + return db + + +def found(db: sqlite3.Connection, *tokens: str) -> set[str]: + """The places the tokens find, joined with OR as a caller would.""" + parts = [match_sql(token) for token in tokens] + sql = " OR ".join(sql for sql, _ in parts) + params = [p for _, params in parts for p in params] + return {row[0] for row in db.execute(f"SELECT place FROM roles WHERE {sql}", params)} + + +# ------------------------------------------------------------------- folding + + +@pytest.mark.parametrize("text, folded", [ + ("Skåne län", "skane lan"), + ("København K", "kobenhavn k"), + ("Kgs. Lyngby", "kgs lyngby"), + ("Malmö", "malmo"), + ("Åbyhøj", "abyhoj"), + ("Ærø", "aero"), + ("Düsseldorf", "dusseldorf"), + ("Zürich", "zurich"), + ("Wrocław", "wroclaw"), + ("Höje-Taastrup, Denmark", "hoje taastrup denmark"), + (" Aarhus N ", "aarhus n"), + ("", ""), +]) +def test_fold(text: str, folded: str) -> None: + assert fold(text) == folded + + +def test_fold_turns_a_ring_and_umlauts_into_the_plain_letter() -> None: + assert fold("å") == "a" + assert (fold("ä"), fold("ö"), fold("ü")) == ("a", "o", "u") + + +# ------------------------------------------------------------------- parsing + + +def test_parse_reads_the_three_kinds() -> None: + assert parse("country:DK") == Token("country", "DK", "Denmark") + assert parse("region:skane") == Token("region", "skane", "Skåne") + assert parse("place:malmo") == Token("place", "malmo", "Malmö") + + +def test_parse_reads_bare_text_as_a_place() -> None: + assert parse("Skåne") == Token("place", "skane", "Skåne") + assert parse("Kgs. Lyngby").value == "kgs lyngby" + assert parse("Foo: Bar").kind == "place" + + +def test_parse_is_forgiving_about_case_and_names() -> None: + assert parse("Country:dk").value == "DK" + assert parse("country:Denmark").value == "DK" + assert parse("region:Skåne").value == "skane" + assert parse("region:Capital Region of Denmark").value == "hovedstaden" + assert parse("place:København").value == "kobenhavn" + + +def test_a_token_round_trips_through_its_string() -> None: + for text in ("country:DK", "region:oresund", "place:kgs lyngby"): + assert str(parse(text)) == text + + +def test_label_reads_a_chip_from_the_address() -> None: + assert label("country:SE") == "Sweden" + assert label("region:hovedstaden") == "Capital Region of Denmark" + assert label("place:kobenhavn") == "København" + assert label("place:copenhagen") == "Copenhagen" + assert label("place:malmo") == "Malmö" + assert label("place:tromso") == "Tromso" + assert label(parse("place:kgs lyngby")) == "Kgs. Lyngby" + assert label("country:ZZ") == "ZZ" + assert label("region:atlantis") == "atlantis" + + +# ---------------------------------------------------------------------- SQL + + +def test_a_place_is_matched_on_whole_words(table: sqlite3.Connection) -> None: + lund = found(table, "place:lund") + assert lund == {"Lund", "Sweden - Lund"} + assert "Billund, Denmark" not in lund and "Albertslund" not in lund + assert found(table, "place:billund") == {"Billund, Denmark"} + + +def test_a_multi_word_place_is_a_phrase(table: sqlite3.Connection) -> None: + assert found(table, "place:kgs lyngby") == {"Kgs. Lyngby", "Kongens Lyngby"} + assert found(table, "place:capital denmark") == set() + assert found(table, "place:denmark capital") == set() + + +def test_diacritics_and_spellings_meet(table: sqlite3.Connection) -> None: + assert found(table, "place:Skåne") == found(table, "place:skane") + assert "Malmö" in found(table, "place:malmo") and "Malmö" in found(table, "place:Malmö") + assert {"København", "København K"} <= found(table, "place:kobenhavn") + + +COPENHAGENS = {"København", "Copenhagen, Denmark", "København K", "Remote OR Copenhagen"} + + +@pytest.mark.parametrize("spelling", ["copenhagen", "kobenhavn", "kopenhamn", "kopenhagen", + "København", "Köpenhamn"]) +def test_copenhagen_is_every_spelling(table: sqlite3.Connection, spelling: str) -> None: + assert found(table, f"place:{spelling}") >= COPENHAGENS + + +@pytest.mark.parametrize("group", places._ALIAS_GROUPS, ids=lambda g: g[0]) +def test_every_alias_finds_every_other(group: tuple[str, ...]) -> None: + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") + db.executemany("INSERT INTO roles VALUES (?, ?, '')", + [(name, fold(name)) for name in group]) + for spelling in group: + sql, params = match_sql(f"place:{spelling}") + hit = {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)} + assert hit == set(group), spelling + + +def test_no_spelling_belongs_to_two_groups() -> None: + seen: dict[str, str] = {} + for group in places._ALIAS_GROUPS: + for name in group: + assert seen.setdefault(fold(name), group[0]) == group[0], name + + +@pytest.mark.parametrize("pair", [("Munich", "München"), ("Cologne", "Köln"), + ("Vienna", "Wien"), ("Zurich", "Zürich"), + ("Geneva", "Genève")]) +def test_a_city_written_two_ways_is_found_either_way( + table: sqlite3.Connection, pair: tuple[str, str]) -> None: + one, other = pair + hit = found(table, f"place:{one}") + assert hit == found(table, f"place:{other}") + words = {w for place in hit for w in fold(place).split()} + assert {fold(one), fold(other)} <= words + + +def test_aa_and_the_ring_are_one_letter(table: sqlite3.Connection) -> None: + hit = found(table, "place:århus") + assert hit == found(table, "place:aarhus") == { + "Aarhus N, Region Central Jutland, Denmark", "Aarhus C, Denmark", + "Aarhus, Denmark; København, Denmark; Stockholm, Sweden"} + + +def test_a_line_with_several_places_is_found_by_each(table: sqlite3.Connection) -> None: + line = "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" + for tok in ("place:aarhus", "place:kobenhavn", "place:stockholm", "place:copenhagen"): + assert line in found(table, tok), tok + + +def test_several_tokens_are_joined_by_or(table: sqlite3.Connection) -> None: + hit = found(table, "country:DK", "region:skane") + assert {"København", "Malmö", "Lund"} <= hit + assert "Stockholm, Sweden" not in hit + + +def test_a_country_is_its_code(table: sqlite3.Connection) -> None: + hit = found(table, "country:DK") + assert {"København", "Denmark", "Ballerup", "Nakskov", "Odense, REGION_DNK 1852 South Denmark, Denmark"} <= hit + assert "Malmö" not in hit and "Berlin, Germany" not in hit + assert "Auckland, New Zealand" not in hit + + +def test_a_country_is_also_a_line_that_names_it_when_country_is_unset( + table: sqlite3.Connection) -> None: + hit = found(table, "country:DK") + # `country` is EU: a board listing offices in several countries. + assert "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" in hit + # Only cities are named, and one of them is Danish. + assert "Copenhagen; London; Warsaw; Milano" in hit + assert "Copenhagen; Warsaw; New York" in hit + # The same words under a country that is set are that country's business. + assert found(table, "country:SE") >= {"Stockholm, Sweden", "Malmö"} + assert "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" in found(table, "country:SE") + assert "Copenhagen, Denmark" in hit # its own country is DK + + +def test_a_country_name_does_not_reach_a_role_of_another_country() -> None: + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") + db.executemany("INSERT INTO roles VALUES (?, ?, ?)", [ + ("Remote, Denmark", "remote denmark", "GB"), ("Copenhagen", "copenhagen", "DE"), + ("Copenhagen", "copenhagen", "DK"), ("Nowhere, Denmark", "nowhere denmark", "")]) + sql, params = match_sql("country:DK") + assert db.execute(f"SELECT country FROM roles WHERE {sql} ORDER BY country", + params).fetchall() == [("",), ("DK",)] + + +def test_country_eu_is_the_column_alone(table: sqlite3.Connection) -> None: + assert found(table, "country:EU") == {"Aarhus, Denmark; København, Denmark; Stockholm, Sweden", + "Copenhagen; London; Warsaw; Milano"} + + +def test_a_region_is_its_names_and_its_towns(table: sqlite3.Connection) -> None: + skane = found(table, "region:skane") + assert skane == {"Malmö", "Lund", "Sweden - Lund", "Helsingborg", "Höganäs, Sweden", + "Malmö, Skåne County, Sweden", "Skåne län, Sweden"} + assert found(table, "region:Skåne") == skane + + hovedstaden = found(table, "region:hovedstaden") + assert {"København", "København K", "København Ø", "Copenhagen, Denmark", "Ballerup", + "Humlebæk, Denmark", "Kgs. Lyngby", "Kongens Lyngby", "Søborg", "Albertslund", + "Måløv, Capital Region of Denmark, Denmark", "Nordhavn og mulighed for hjemmearbejde", + } <= hovedstaden + assert not hovedstaden & {"Aarhus C, Denmark", "Billund, Denmark", "Malmö", "Nakskov"} + + +def test_zealand_is_the_danish_region_and_not_new_zealand(table: sqlite3.Connection) -> None: + assert found(table, "region:zealand") == {"Kalundborg, Region Zealand, Denmark", "Nakskov"} + + +def test_jutland_and_funen(table: sqlite3.Connection) -> None: + assert {"Billund, Denmark", "Aarhus C, Denmark"} <= found(table, "region:jutland") + assert found(table, "region:funen") == {"Odense, REGION_DNK 1852 South Denmark, Denmark"} + + +def test_oresund_is_the_two_capital_regions(table: sqlite3.Connection) -> None: + assert found(table, "region:oresund") == ( + found(table, "region:hovedstaden") | found(table, "region:skane")) + + +def test_a_region_is_a_place_written_as_bare_text(table: sqlite3.Connection) -> None: + assert found(table, "Skåne") == found(table, "region:skane") + assert found(table, "place:scania") == found(table, "region:skane") + assert found(table, "Denmark") == found(table, "country:DK") + + +def test_a_token_that_names_nothing_finds_nothing(table: sqlite3.Connection) -> None: + assert found(table, "region:atlantis") == set() + assert found(table, "place:") == set() + assert found(table, "") == set() + assert found(table, "place:nowhere-in-particular") == set() + + +def test_the_columns_are_the_callers(table: sqlite3.Connection) -> None: + sql, params = match_sql("country:DK", place="r.pf", country="r.c") + assert "r.pf" in sql and "r.c " in sql and "place_fold" not in sql + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE r (pf TEXT, c TEXT)") + db.executemany("INSERT INTO r VALUES (?, ?)", [("lund", "SE"), ("billund", "DK")]) + sql, params = match_sql("place:lund", place="r.pf", country="r.c") + assert db.execute(f"SELECT pf FROM r WHERE {sql}", params).fetchall() == [("lund",)] + + +def test_sql_is_parenthesised_and_has_one_parameter_per_question_mark() -> None: + for token in ("country:DK", "region:oresund", "place:copenhagen", "place:denmark", + "region:atlantis"): + sql, params = match_sql(token) + assert sql.startswith("(") and sql.endswith(")") + assert sql.count("?") == len(params), token + + +def test_a_wildcard_in_a_value_is_a_letter() -> None: + assert places._escape("50%_off\\") == "50\\%\\_off\\\\" + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") + db.executemany("INSERT INTO roles VALUES (?, ?, '')", [("a b", "a b"), ("axb", "axb"), + ("a_b", "a_b")]) + for token in ("place:a_b", "place:a%b"): + sql, params = match_sql(token) + assert {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)} \ + == {"a b"}, token + # A column that is not folded still matches a literal underscore, not any character. + sql, params = places._words("place", ["a_b"]) + assert db.execute(f"SELECT place FROM roles WHERE {sql}", params).fetchall() == [("a_b",)] + + +# ----------------------------------------------------------------- the tables + + +def test_every_region_has_names_and_towns() -> None: + assert set(REGIONS) == {"skane", "hovedstaden", "zealand", "oresund", "jutland", "funen"} + for key, region in REGIONS.items(): + assert region.key == key and region.label and region.names and region.towns + + +def test_the_bare_word_zealand_is_no_name_since_new_zealand_has_it() -> None: + assert "zealand" not in places._PHRASES["zealand"] + + +def test_the_capital_and_scanian_towns_are_in_their_regions() -> None: + skane = {fold(t) for t in REGIONS["skane"].towns} + assert {"malmo", "lund", "helsingborg", "kristianstad", "landskrona", "trelleborg", + "angelholm", "hassleholm", "ystad", "eslov", "hoganas", "staffanstorp", "lomma", + "bjuv", "burlov", "kavlinge", "svedala", "vellinge"} <= skane + hoved = {fold(t) for t in REGIONS["hovedstaden"].towns} + assert {"kobenhavn", "kgs lyngby", "soborg", "ballerup", "malov", "bagsvaerd", "humlebaek", + "smorum", "nordhavn", "frederiksberg", "hillerod", "helsingor"} <= hoved + + +# ----------------------------------------------------------------- suggestions + +PLACES = [ + ("København", "DK", 69), ("Copenhagen, Denmark", "DK", 56), ("København K", "DK", 35), + ("København Ø", "DK", 34), ("Kgs. Lyngby", "DK", 16), ("Billund, Denmark", "DK", 18), + ("Aarhus N, Region Central Jutland, Denmark", "DK", 26), ("Albertslund", "DK", 10), + ("Denmark", "DK", 27), ("Remote OR Copenhagen", "DK", 1), + ("Malmö", "SE", 214), ("Lund", "SE", 127), ("Sweden - Lund", "SE", 29), + ("Malmö, Skåne County, Sweden", "SE", 3), ("Stockholm, Sweden", "SE", 287), + ("Aarhus, Denmark; København, Denmark; Stockholm, Sweden", "EU", 1), + ("Remote", "", 40), ("Hybrid", "", 12), ("Europe", "EU", 20), ("", "", 5), + ("Berlin, Germany", "DE", 100), ("Auckland, New Zealand", "NZ", 25), + ("The Hague, Netherlands", "NL", 9), ("Den Haag", "NL", 4), +] + + +def tokens(q: str, **kw) -> list[str]: + return [s.token for s in suggest(q, PLACES, **kw)] + + +def test_skane_is_suggested_however_it_is_typed() -> None: + for q in ("Skåne", "skane", "SKANE", "ska", "Scania", "scan"): + assert "region:skane" in tokens(q), q + hit = next(s for s in suggest("skane", PLACES) if s.token == "region:skane") + assert hit == Suggestion("region:skane", "Skåne", "region", 214 + 127 + 29 + 3) + + +def test_a_country_counts_the_roles_that_carry_its_code() -> None: + hit = suggest("den", PLACES)[0] + assert hit == Suggestion("country:DK", "Denmark", "country", + 69 + 56 + 35 + 34 + 16 + 18 + 26 + 10 + 27 + 1) + assert tokens("swe")[0] == "country:SE" + assert tokens("sverige")[0] == "country:SE" + assert tokens("danmark")[0] == "country:DK" + + +def test_countries_come_first_then_regions_then_towns() -> None: + kinds = [s.kind for s in suggest("den", PLACES)] + assert kinds == sorted(kinds, key=["country", "region", "place"].index) + assert kinds[0] == "country" and "region" in kinds and kinds[-1] == "place" + + +def test_towns_are_grouped_and_labelled_by_their_commonest_spelling() -> None: + lund = [s for s in suggest("lund", PLACES) if s.kind == "place"] + assert lund == [Suggestion("place:lund", "Lund", "place", 127 + 29)] + kbh = [s for s in suggest("kob", PLACES) if s.kind == "place"] + assert kbh[0].label == "København" and kbh[0].token == "place:kobenhavn" + + +def test_a_postal_district_is_not_a_town_of_its_own() -> None: + labels = [s.label for s in suggest("kob", PLACES) if s.kind == "place"] + assert "København K" not in labels and "København Ø" not in labels + assert [s.label for s in suggest("aarhus", PLACES) if s.kind == "place"] == ["Aarhus"] + + +def test_spellings_of_one_town_are_one_suggestion() -> None: + towns = [s for s in suggest("cop", PLACES) if s.kind == "place"] + assert len(towns) == 1 + # København, K and Ø, the one in a list of three towns; Copenhagen, and "Remote OR Copenhagen". + assert towns[0].label == "København" and towns[0].count == 69 + 35 + 34 + 1 + 56 + 1 + assert [s.token for s in suggest("kobenhavn", PLACES) if s.kind == "place"] \ + == [towns[0].token] + + +def test_only_the_first_comma_part_of_each_semicolon_part_is_a_town() -> None: + labels = {s.label for q in "abcdefghijklmnopqrstuvwxyz" + for s in suggest(q, PLACES, limit=999) if s.kind == "place"} + assert {"Aarhus", "Stockholm", "Malmö", "Lund", "Berlin"} <= labels + assert not {"Denmark", "Sweden", "Region Central Jutland", "Remote", "Hybrid", "Europe"} \ + & labels + + +def test_remote_hybrid_and_country_names_are_no_town() -> None: + for q in ("rem", "hyb", "eur", "denm"): + assert [s for s in suggest(q, PLACES) if s.kind == "place"] == [], q + + +def test_a_word_prefix_finds_a_multi_word_town() -> None: + assert tokens("haag") == tokens("hague") == tokens("den haag") == tokens("the hag") \ + == ["place:the hague"] + assert tokens("kgs ly") == ["place:kgs lyngby"] + assert tokens("lyngby") == ["place:kgs lyngby"] + + +def test_the_word_must_start_not_end() -> None: + assert "place:lund" not in tokens("und") + assert "place:billund" not in tokens("lund") + assert "place:albertslund" not in tokens("lund") + + +def test_new_zealand_is_a_country_and_zealand_a_region_only_when_it_has_roles() -> None: + assert tokens("zea")[:1] == ["country:NZ"] + assert "region:zealand" not in tokens("zea") # no Zealand role in PLACES + assert "country:AU" not in tokens("aus") # no role of its own + + +def test_limit_and_the_empty_query() -> None: + assert suggest("", PLACES) == [] and suggest(" ", PLACES) == [] and suggest("---", PLACES) == [] + assert len(suggest("a", PLACES, limit=3)) == 3 + assert len(suggest("a", PLACES)) <= 12 + + +def test_towns_are_ordered_by_roles() -> None: + towns = [s for s in suggest("l", PLACES) if s.kind == "place"] + assert [s.count for s in towns] == sorted((s.count for s in towns), reverse=True) + + +def test_suggest_takes_any_iterable() -> None: + assert suggest("lund", iter(PLACES)) == suggest("lund", PLACES) + + +def test_a_suggested_token_finds_the_roles_it_counted(table: sqlite3.Connection) -> None: + for s in suggest("kob", PLACES) + suggest("lund", PLACES) + suggest("skane", PLACES): + assert parse(s.token).kind == s.kind + assert found(table, s.token), s.token + assert label(s.token) == s.label or s.kind == "place" From 1898f2a8a6263b9664ed52f03cd4c4951d6644e5 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 22:04:21 +0200 Subject: [PATCH 2/2] Match places with a registered SQL function, suggest regions on their own names, fold lines that hold a town into it Co-Authored-By: Claude --- app/places.py | 245 ++++++++++---- tests/test_places.py | 788 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 947 insertions(+), 86 deletions(-) diff --git a/app/places.py b/app/places.py index ae564a6..0505189 100644 --- a/app/places.py +++ b/app/places.py @@ -13,23 +13,29 @@ of three tokens, and several tokens are joined by OR: place:malmo the town's fold and its aliases, so `copenhagen`, `kobenhavn` and `kopenhamn` find each other -`match_sql` turns a token into SQL over `roles.place_fold` and `roles.country`; +`matches` answers whether one role's folded place and country fit a token, in +Python: a region is over a hundred phrases, and as `LIKE` clauses that costs +several hundred milliseconds a query, as a set intersection on the words of a +line it costs a few. `register` gives a connection that answer as the SQL +function `place_match`, and `match_sql` returns the clause that calls it. `suggest` turns typed text into tokens for the autocomplete; `label` reads a -chip's text back from the address. No database access here, the caller hands -over the places it holds. +chip's text back from the address. This module opens no database; the caller +hands over the places it holds, or its connection. """ from __future__ import annotations import re +import sqlite3 from collections import Counter from collections.abc import Iterable from dataclasses import dataclass +from functools import lru_cache from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold -__all__ = ["REGIONS", "Region", "Suggestion", "Token", "fold", "label", "match_sql", - "parse", "suggest"] +__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known", + "label", "match_sql", "matches", "parse", "register", "suggest"] def fold(text: str) -> str: @@ -202,14 +208,16 @@ def _expand(text: str) -> tuple[str, ...]: class Region: """A region below a country, or across two. `names` are what a board calls the region itself, `towns` the places in it; a line is in the region when it - has one of either as whole words. Spellings are shown with their marks and - folded when used.""" + has one of either as whole words, or is in a region it `includes`, by key. + Only `names` are what the region is suggested and read as. Spellings are + shown with their marks and folded when used.""" key: str label: str names: tuple[str, ...] towns: tuple[str, ...] country: str = "" + includes: tuple[str, ...] = () def _split(text: str) -> tuple[str, ...]: @@ -281,12 +289,12 @@ _FUNEN = Region( Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""), "DK") -# The Øresund region is the two capital regions either side of the sound. +# The Øresund region is the two capital regions either side of the sound. It is +# suggested on its own names only; a line in either region is in it. _ORESUND = Region( "oresund", "Øresund", - _split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen") - + _HOVEDSTADEN.names + _SKANE.names, - _HOVEDSTADEN.towns + _SKANE.towns) + _split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"), + (), includes=("hovedstaden", "skane")) REGIONS: dict[str, Region] = { r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)} @@ -294,11 +302,13 @@ REGIONS: dict[str, Region] = { def _phrases(region: Region) -> tuple[str, ...]: """Every folded phrase that puts a line in the region, with the other - spellings of its towns.""" + spellings of its towns, and those of the regions it includes.""" found: dict[str, None] = {} for text in (*region.names, *region.towns): for form in _expand(fold(text)): found[form] = None + for key in region.includes: + found.update(dict.fromkeys(_phrases(REGIONS[key]))) return tuple(found) @@ -321,6 +331,24 @@ for _region in REGIONS.values(): _NOT_A_TOWN |= frozenset(_REGION_BY_NAME) +def _group(folded: str) -> str: + """The key of the town a fold is: the first spelling of its alias group, else + the fold with `aa` as `a`.""" + return _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1] + + +def _known_towns() -> dict[str, str]: + """Every town the tables name, as a fold, to its group: the alias groups, the + towns of the regions and the cities `country_of` knows.""" + names = [*_GROUPS, *(fold(city) for city in _CITIES), + *(fold(town) for region in REGIONS.values() for town in region.towns)] + return {form: _group(form) for name in names for form in _variants(name)} + + +_KNOWN = _known_towns() +_KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN) + + # --------------------------------------------------------------------- tokens @@ -369,57 +397,118 @@ def label(token: str | Token) -> str: return (token if isinstance(token, Token) else parse(token)).label -# ------------------------------------------------------------------------ SQL +# ------------------------------------------------------------------- matching -_NEVER = ("0", []) +SQL_FUNCTION = "place_match" -def _escape(word: str) -> str: - """A folded word holds no wildcard, but a caller may hand over raw text.""" - return re.sub(r"([\\%_])", r"\\\1", word) +def known(token: str) -> bool: + """Whether the token names something: a country of the table (or `EU`), a + region of `REGIONS`, a place with a value. `region:atlantis`, `country:ZZ` + and `place:` are not.""" + return _known(parse(token)) -def _words(column: str, phrases: Iterable[str]) -> tuple[str, list]: - """Any of the phrases as whole words of the column. A multi-word phrase is - a phrase: `kgs lyngby` needs both words, side by side.""" - phrases = [p for p in dict.fromkeys(phrases) if p] - if not phrases: - return _NEVER - clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'" - return " OR ".join([clause] * len(phrases)), [f"% {_escape(p)} %" for p in phrases] +def _known(tok: Token) -> bool: + if tok.kind == "country": + return tok.value in _COUNTRY_LABELS or tok.value == "EU" + if tok.kind == "region": + return tok.value in REGIONS + return bool(tok.value) -def _country(code: str, place: str, country: str) -> tuple[str, list]: - """The role's own country, or a line that names the country while `country` - is empty or `EU`: a board listing several offices has no single country.""" - names, params = _words(place, _COUNTRY_MENTIONS.get(code, ())) - if not params: - return f"{country} = ?", [code] - return f"{country} = ? OR ({country} IN ('', 'EU') AND ({names}))", [code, *params] +@dataclass(frozen=True) +class _Matcher: + """What a token asks of a role. `phrases` are whole-word phrases of the + folded place; `countries` are `(code, mentions)`: the role's own country, or + a line with one of the mentions while `country` is empty or `EU`, since a + board listing several offices has no single country.""" + + phrases: frozenset[str] + countries: tuple[tuple[str, frozenset[str]], ...] + longest: int + + +@lru_cache(maxsize=1024) +def _matcher(token: str) -> _Matcher | None: + """The token read once: `None` when it names nothing. + + A `place:` value that is also a region or a country is matched as that too: + `place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate + written as a bare word means what the owner meant by it.""" + tok = parse(token) + if not _known(tok): + return None + phrases: set[str] = set() + countries: list[tuple[str, frozenset[str]]] = [] + if tok.kind == "country": + countries.append((tok.value, frozenset(_COUNTRY_MENTIONS.get(tok.value, ())))) + elif tok.kind == "region": + phrases.update(_PHRASES[tok.value]) + else: + phrases.update(_expand(tok.value)) + phrases.update(_PHRASES.get(_REGION_BY_NAME.get(tok.value, ""), ())) + if code := _COUNTRY_BY_NAME.get(tok.value): + countries.append((code, frozenset(_COUNTRY_MENTIONS.get(code, ())))) + phrases.discard("") + longest = max((len(p.split()) for p in (*phrases, *(m for _, ms in countries for m in ms))), + default=1) + return _Matcher(frozenset(phrases), tuple(countries), longest) + + +@lru_cache(maxsize=32768) +def _runs(place_fold: str, longest: int) -> frozenset[str]: + """Every run of up to `longest` words of a folded line, each a whole-word + phrase the line holds.""" + words = place_fold.split() + return frozenset(" ".join(words[i:i + n]) + for n in range(1, longest + 1) for i in range(len(words) - n + 1)) + + +def matches(place_fold: str, country: str, token: str) -> bool: + """Whether a role whose folded place and country are these is found by the + token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or + a Danish city) while `country` is empty or `EU`. `region:skane` is any of + its names or towns as whole words, `place:malmo` the town and its aliases. + A token that names nothing (`known` is false) matches no role.""" + match = _matcher(token) + if match is None: + return False + runs = None + if match.phrases: + runs = _runs(place_fold, match.longest) + if not match.phrases.isdisjoint(runs): + return True + for code, mentions in match.countries: + if country == code: + return True + if mentions and country in ("", "EU"): + runs = _runs(place_fold, match.longest) if runs is None else runs + if not mentions.isdisjoint(runs): + return True + return False + + +def _place_match(place_fold: str | None, country: str | None, token: str) -> int: + return int(matches(place_fold or "", country or "", token)) + + +def register(conn: sqlite3.Connection) -> None: + """Give a connection `place_match(place_fold, country, token)`, which is 1 when + `matches` is true. `match_sql` returns a clause that calls it.""" + conn.create_function(SQL_FUNCTION, 3, _place_match, deterministic=True) def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \ -> tuple[str, list]: - """SQL over the two columns for one token, parenthesised, with `?` parameters. - Several tokens are joined with OR by the caller. Bare text is a place. - - A `place:` value that is also a region or a country is matched as that too: - `place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate - written as a bare word means what the owner meant by it. A token that names - nothing known (`region:atlantis`, an empty value) matches no role.""" + """SQL over the two columns for one token, with one `?` parameter. It calls + `place_match`, so the connection needs `register`. Several tokens are joined + with OR by the caller. Bare text is a place. A token that names nothing + (`known` is false) is `0`, which matches no role.""" tok = parse(token) - if tok.kind == "country": - sql, params = _country(tok.value, place, country) - elif tok.kind == "region": - sql, params = _words(place, _PHRASES.get(tok.value, ())) - else: - region = _REGION_BY_NAME.get(tok.value) - sql, params = _words(place, [*_expand(tok.value), *_PHRASES.get(region, ())]) \ - if tok.value else _NEVER - if tok.value in _COUNTRY_BY_NAME: - more, extra = _country(_COUNTRY_BY_NAME[tok.value], place, country) - sql, params = f"{sql} OR {more}", [*params, *extra] - return f"({sql})", params + if not _known(tok): + return "0", [] + return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)] # ---------------------------------------------------------------- suggestions @@ -442,9 +531,9 @@ def _prefixes(q: str, candidates: Iterable[str]) -> bool: return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates) -#: Where one town ends in a line: `;` between places, and `or` / `eller` / `og` -#: between two towns in one ("København V eller Aalborg"). -_PART = re.compile(r"\s*;\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE) +#: Where one town ends in a line: `;` and `|` between places, and `or` / `eller` / +#: `og` between two towns in one ("København V eller Aalborg"). +_PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE) #: Danish boards append "and the possibility of working from home". _AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE) @@ -471,16 +560,32 @@ def _towns(place: str) -> list[str]: return found +def _towns_in(folded: str) -> list[str]: + """The known towns a fold holds as whole words, longest first from the left, + as their groups: `munich berlin` is Munich and Berlin, `copenhagen office` is + Copenhagen, `hoje taastrup` is one town and not also Taastrup.""" + words = folded.split() + found: list[str] = [] + i = 0 + while i < len(words): + for n in range(min(_KNOWN_LONGEST, len(words) - i), 0, -1): + if (group := _KNOWN.get(" ".join(words[i:i + n]))) is not None: + found.append(group) + i += n + break + else: + i += 1 + return list(dict.fromkeys(found)) + + def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]: """Open roles per region, over folded lines. A line is cut into every run of words once, and each region asks whether it holds one of its phrases.""" - sets = {key: set(_PHRASES[key]) for key in keys} - longest = max((len(p.split()) for key in sets for p in sets[key]), default=1) + sets = {key: frozenset(_PHRASES[key]) for key in keys} + longest = max((len(p.split()) for phrases in sets.values() for p in phrases), default=1) counts: Counter[str] = Counter() for text, roles in places: - words = text.split() - runs = {" ".join(words[i:i + n]) - for n in range(1, longest + 1) for i in range(len(words) - n + 1)} + runs = _runs(text, longest) for key, phrases in sets.items(): if not phrases.isdisjoint(runs): counts[key] += roles @@ -516,20 +621,32 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \ regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key]) for key in named if counts[key]] + # A line's town is a group. A line that is a town is that group's spelling; + # one that only holds a known town ("Copenhagen Office", "Munich - Berlin") + # counts under each such town and shows as none; anything else is its own. + counts_by_town: Counter[str] = Counter() spellings: dict[str, Counter[str]] = {} written: dict[str, set[str]] = {} for place, _, roles in rows: for town in _towns(place): folded = fold(town) - group = _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1] + if folded not in _KNOWN and (inside := _towns_in(folded)): + for group in inside: + counts_by_town[group] += roles + continue + group = _group(folded) + counts_by_town[group] += roles spellings.setdefault(group, Counter())[town] += roles written.setdefault(group, set()).add(folded) towns = [] - for group, seen in spellings.items(): - if not _prefixes(text, [form for folded in written[group] for form in _expand(folded)]): + for group, total in counts_by_town.items(): + forms = {group, *written.get(group, ())} + if not _prefixes(text, [form for folded in forms for form in _expand(folded)]): continue - shown = min(seen, key=lambda s: (-seen[s], s)) - towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", sum(seen.values()))) + seen = spellings.get(group) + shown = min(seen, key=lambda s: (-seen[s], s)) if seen \ + else _TOWN_LABELS.get(group) or group.title() + towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total)) countries.sort(key=lambda s: (-s.count, s.label)) regions.sort(key=lambda s: (-s.count, s.label)) diff --git a/tests/test_places.py b/tests/test_places.py index d1f83df..98e3395 100644 --- a/tests/test_places.py +++ b/tests/test_places.py @@ -3,6 +3,7 @@ The rows are `place` strings as boards write them, taken from the live database. Each token's SQL runs against an in-memory table shaped like `roles`.""" +# ruff: noqa: RUF001 (real place strings, with the letters boards write) from __future__ import annotations import sqlite3 @@ -10,7 +11,8 @@ import sqlite3 import pytest from app import places -from app.places import REGIONS, Suggestion, Token, fold, label, match_sql, parse, suggest +from app.places import (REGIONS, Suggestion, Token, fold, known, label, match_sql, matches, + parse, register, suggest) #: (place, country) as a board wrote them and `country_of` read them. ROWS = [ @@ -37,9 +39,15 @@ ROWS = [ ] +def connect() -> sqlite3.Connection: + db = sqlite3.connect(":memory:") + register(db) + return db + + @pytest.fixture() def table() -> sqlite3.Connection: - db = sqlite3.connect(":memory:") + db = connect() db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") db.executemany("INSERT INTO roles VALUES (?, ?, ?)", [(place, fold(place), country) for place, country in ROWS]) @@ -153,7 +161,7 @@ def test_copenhagen_is_every_spelling(table: sqlite3.Connection, spelling: str) @pytest.mark.parametrize("group", places._ALIAS_GROUPS, ids=lambda g: g[0]) def test_every_alias_finds_every_other(group: tuple[str, ...]) -> None: - db = sqlite3.connect(":memory:") + db = connect() db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") db.executemany("INSERT INTO roles VALUES (?, ?, '')", [(name, fold(name)) for name in group]) @@ -223,7 +231,7 @@ def test_a_country_is_also_a_line_that_names_it_when_country_is_unset( def test_a_country_name_does_not_reach_a_role_of_another_country() -> None: - db = sqlite3.connect(":memory:") + db = connect() db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") db.executemany("INSERT INTO roles VALUES (?, ?, ?)", [ ("Remote, Denmark", "remote denmark", "GB"), ("Copenhagen", "copenhagen", "DE"), @@ -261,6 +269,15 @@ def test_jutland_and_funen(table: sqlite3.Connection) -> None: assert found(table, "region:funen") == {"Odense, REGION_DNK 1852 South Denmark, Denmark"} +def test_oresund_finds_a_line_of_either_region_but_is_named_by_its_own_words() -> None: + assert matches("lund sweden", "SE", "region:oresund") + assert matches("kobenhavn k", "DK", "region:oresund") + assert not matches("aarhus", "DK", "region:oresund") + assert "skane" not in {fold(n) for n in REGIONS["oresund"].names} + assert REGIONS["oresund"].includes == ("hovedstaden", "skane") + assert parse("region:Capital Region of Denmark").value == "hovedstaden" + + def test_oresund_is_the_two_capital_regions(table: sqlite3.Connection) -> None: assert found(table, "region:oresund") == ( found(table, "region:hovedstaden") | found(table, "region:skane")) @@ -279,27 +296,28 @@ def test_a_token_that_names_nothing_finds_nothing(table: sqlite3.Connection) -> assert found(table, "place:nowhere-in-particular") == set() -def test_the_columns_are_the_callers(table: sqlite3.Connection) -> None: +def test_the_columns_are_the_callers() -> None: sql, params = match_sql("country:DK", place="r.pf", country="r.c") - assert "r.pf" in sql and "r.c " in sql and "place_fold" not in sql - db = sqlite3.connect(":memory:") + assert sql == "place_match(r.pf, r.c, ?)" and params == ["country:DK"] + db = connect() db.execute("CREATE TABLE r (pf TEXT, c TEXT)") db.executemany("INSERT INTO r VALUES (?, ?)", [("lund", "SE"), ("billund", "DK")]) sql, params = match_sql("place:lund", place="r.pf", country="r.c") assert db.execute(f"SELECT pf FROM r WHERE {sql}", params).fetchall() == [("lund",)] -def test_sql_is_parenthesised_and_has_one_parameter_per_question_mark() -> None: - for token in ("country:DK", "region:oresund", "place:copenhagen", "place:denmark", - "region:atlantis"): +def test_sql_has_one_parameter_per_question_mark_and_a_token_is_normalised() -> None: + for token in ("country:DK", "region:oresund", "place:copenhagen", "place:denmark", "Skåne"): sql, params = match_sql(token) - assert sql.startswith("(") and sql.endswith(")") - assert sql.count("?") == len(params), token + assert sql.startswith("place_match(") and sql.count("?") == len(params) == 1, token + assert match_sql("country:dk")[1] == ["country:DK"] + assert match_sql("Kgs. Lyngby")[1] == ["place:kgs lyngby"] + for token in ("region:atlantis", "country:ZZ", "place:", ""): + assert match_sql(token) == ("0", []), token def test_a_wildcard_in_a_value_is_a_letter() -> None: - assert places._escape("50%_off\\") == "50\\%\\_off\\\\" - db = sqlite3.connect(":memory:") + db = connect() db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") db.executemany("INSERT INTO roles VALUES (?, ?, '')", [("a b", "a b"), ("axb", "axb"), ("a_b", "a_b")]) @@ -307,9 +325,26 @@ def test_a_wildcard_in_a_value_is_a_letter() -> None: sql, params = match_sql(token) assert {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)} \ == {"a b"}, token - # A column that is not folded still matches a literal underscore, not any character. - sql, params = places._words("place", ["a_b"]) - assert db.execute(f"SELECT place FROM roles WHERE {sql}", params).fetchall() == [("a_b",)] + + +def test_known_is_false_for_a_token_that_names_nothing() -> None: + for token in ("region:atlantis", "country:ZZ", "place:", "region:", "country:", "", ":"): + assert not known(token), token + for token in ("country:DK", "country:EU", "country:dk", "region:skane", "region:Skåne", + "place:nowhere-in-particular", "Skåne", "country:Denmark"): + assert known(token), token + + +def test_matches_is_false_for_a_token_that_names_nothing() -> None: + assert not matches("copenhagen", "DK", "region:atlantis") + assert not matches("copenhagen", "ZZ", "country:ZZ") + assert not matches("copenhagen", "DK", "place:") + + +def test_null_columns_match_nothing_and_do_not_raise() -> None: + db = connect() + assert db.execute("SELECT place_match(NULL, NULL, 'place:lund')").fetchone() == (0,) + assert db.execute("SELECT place_match(NULL, 'DK', 'country:DK')").fetchone() == (1,) # ----------------------------------------------------------------- the tables @@ -318,7 +353,8 @@ def test_a_wildcard_in_a_value_is_a_letter() -> None: def test_every_region_has_names_and_towns() -> None: assert set(REGIONS) == {"skane", "hovedstaden", "zealand", "oresund", "jutland", "funen"} for key, region in REGIONS.items(): - assert region.key == key and region.label and region.names and region.towns + assert region.key == key and region.label and region.names + assert region.towns or region.includes def test_the_bare_word_zealand_is_no_name_since_new_zealand_has_it() -> None: @@ -348,6 +384,11 @@ PLACES = [ ("Remote", "", 40), ("Hybrid", "", 12), ("Europe", "EU", 20), ("", "", 5), ("Berlin, Germany", "DE", 100), ("Auckland, New Zealand", "NZ", 25), ("The Hague, Netherlands", "NL", 9), ("Den Haag", "NL", 4), + ("Copenhagen Office", "DK", 16), ("Copenhagen Hub - Løvstræde", "DK", 14), + ("DK-Copenhagen-MSO", "DK", 2), + ("Munich", "DE", 40), ("München", "DE", 10), ("Munich Office", "DE", 5), + ("Munich (DE)", "DE", 3), ("Munich - Berlin", "DE", 2), + ("Garching bei München (Munich)", "DE", 4), ("Kalundborg, Region Zealand, Denmark", "DK", 14), ] @@ -365,7 +406,8 @@ def test_skane_is_suggested_however_it_is_typed() -> None: def test_a_country_counts_the_roles_that_carry_its_code() -> None: hit = suggest("den", PLACES)[0] assert hit == Suggestion("country:DK", "Denmark", "country", - 69 + 56 + 35 + 34 + 16 + 18 + 26 + 10 + 27 + 1) + 69 + 56 + 35 + 34 + 16 + 18 + 26 + 10 + 27 + 1 + + 16 + 14 + 2 + 14) assert tokens("swe")[0] == "country:SE" assert tokens("sverige")[0] == "country:SE" assert tokens("danmark")[0] == "country:DK" @@ -394,7 +436,7 @@ def test_spellings_of_one_town_are_one_suggestion() -> None: towns = [s for s in suggest("cop", PLACES) if s.kind == "place"] assert len(towns) == 1 # København, K and Ø, the one in a list of three towns; Copenhagen, and "Remote OR Copenhagen". - assert towns[0].label == "København" and towns[0].count == 69 + 35 + 34 + 1 + 56 + 1 + assert towns[0].label == "København" and towns[0].count == 69 + 35 + 34 + 1 + 56 + 1 + 16 + 14 + 2 assert [s.token for s in suggest("kobenhavn", PLACES) if s.kind == "place"] \ == [towns[0].token] @@ -425,9 +467,10 @@ def test_the_word_must_start_not_end() -> None: assert "place:albertslund" not in tokens("lund") -def test_new_zealand_is_a_country_and_zealand_a_region_only_when_it_has_roles() -> None: +def test_a_country_or_region_with_no_open_role_is_not_suggested() -> None: assert tokens("zea")[:1] == ["country:NZ"] - assert "region:zealand" not in tokens("zea") # no Zealand role in PLACES + assert "region:zealand" in tokens("zea") # Kalundborg + assert "region:funen" not in tokens("fun") # no Funen role in PLACES assert "country:AU" not in tokens("aus") # no role of its own @@ -451,3 +494,704 @@ def test_a_suggested_token_finds_the_roles_it_counted(table: sqlite3.Connection) assert parse(s.token).kind == s.kind assert found(table, s.token), s.token assert label(s.token) == s.label or s.kind == "place" + + +# ------------------------------------------------- the function against LIKE + +#: `countryplace` of open roles on the live database, distinct: a random +#: sample plus, for each token below, lines that hold it. Some lines have no country. +REAL = """ +NL 's-Hertogenbosch, Netherlands + 3 Locations + 5 Locations +DK Aabenraa eller København +DK Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position +DK Aalborg, Denmark; Aarhus, Denmark; Copenhagen, Denmark; Odense, Denmark; Søborg, Denmark +DK Aarhus C, Denmark +DK Aarhus N, Denmark +DK Aarhus or Valby +EU Aarhus, Denmark; København, Denmark; Stockholm, Sweden + Aix-en-Provence +DK Allerød or Stockholm (SE) +DK Allerød, Copenhagen, or Stockholm, Hybrid position +KZ Almaty, Almatı oblısı, Kazakhstan +EU Amsterdam, Netherlands; Berlin, Germany; København, Denmark; Vilnius, Lithuania +EU Amsterdam, Netherlands; Berlin, Germany; Munich, Germany; Remote, Germany +EU Amsterdam, Netherlands; Manchester, United Kingdom +EU Amsterdam; Germany +FR Angers, France +DE Ansbach, Mittelfranken, Germany +MX Apaseo el Grande, GUANAJUATO, Mexico +US Arizona; California; Washington +NL Arnhem, GELDERLAND, Netherlands +BE Asse, , Belgium + Athens, Greece; Cyprus; Frankfurt am Main, Germany; Munich, Germany; Tel Aviv, Israel + Athens, Greece; London, United Kingdom; Tel Aviv, Israel + Auckland +US Austin; New York City; Palo Alto +AU Australian Capital Territory, Australia +DK Bad Homburg; Copenhagen +DK Bagsværd +DK Ballerup or Aarhus V +DK Ballerup, , Denmark +DK Ballerup, Ballerup +DK Ballerup, Denmark +IN Bangalore, KA, India +CN Bao Ding, Hebei, China +ES Barcelona, Catalonia +EU Barcelona, Spain; Stockholm, Stockholm County, Sweden +EU Barcelona; Berlin; London; Munich; Paris +EU Barcelona; Geneva +DE Barsinghausen, Germany +PL Bartoszyce, Warminsko-mazurskie, Poland +MY Bayan lepas, Pulau Pinang, Malaysia +EU Belgium; Netherlands +RS Belgrade - Savski Venac, Serbia +EU Belgrade, Serbia; Berlin, Germany; Munich, Germany +US Bellevue, United States of America +US Bellevue, WA, United States +US Bellevue, Washington; Denver, Colorado; Seattle, Washington; Washington, D.C. +IN Bengaluru, India +RS Beograd, , Serbia +NO Bergen, , Norway +NL Berkel en Rodenrijs, ROTTERDAM, Netherlands +DE Berlin Office; Munich Office +DE Berlin Office; Munich Office; Remotely in Germany +DE Berlin, Berlin, Germany +DE Berlin, Germany +EU Berlin, Germany; Amsterdam, Netherlands; Copenhagen, Denmark +EU Berlin, Germany; Copenhagen, Denmark +EU Berlin, Germany; London, United Kingdom +EU Berlin, Germany; London, United Kingdom; Stockholm, Sweden +EU Berlin, Germany; Munich, Germany; Prague, Czech Republic +EU Berlin, Germany; Paris, France +DE Berlin; Cologne; Frankfurt; Hamburg; Munich; Stuttgart +DE Berlin; Frankfurt; Munich +DE Berlin; Frankfurt; Munich; Stuttgart +EU Berlin; London; Munich +EU Berlin; London; Munich; Paris +EU Berlin; London; Munich; Paris; Stockholm; Tallinn +DE Berlin; Munich +CH Bern, Switzerland +NL Best, Noord-Brabant, Netherlands +DE Bietigheim-Bissingen, Baden-Württemberg, Germany +DK Billund Kornmarken, Denmark +DK Billund, Denmark +DK Billund, København +DK Billund, København K +DK Billund; Copenhagen Hub - Løvstræde +EU Billund; London; Copenhagen Hub - Løvstræde +DK Birkerød +SE Bjuv +DK Bjæverskov +DE Bochum, NW, Germany +DE Bonn, Germany +FR Bordeaux +JP Bosch Corporation_Shibuya_BGSJ, Tokyo, Japan +US Boston, MA, United States +US Bozeman, MT, United States +DK Brabrand, Denmark + Brest +US Bridgewater, United States of America +FR Briey, France +GB Bristol, UK; London, UK +SI Brnik - Aerodrom, , Slovenia +DK Bronshoj, Denmark +BE Bruxelles, Bruxelles, Belgium +DK Brøndby +DK Brøndby eller Fredericia +DK Brøndby eller Fredericia og mulighed for hjemmearbejde +US Burlington, United States + Busan +DE Butzbach, HE, Germany +RO Buzau, Com Caragele, Constanta, Romania +SE Båstad +DE Böblingen, Baden-Württemberg, Germany +CA CA-Toronto +AU Camberwell, Australia +GB Carlisle, England, United Kingdom +DE Carlsfelder Str. 3, 06188 Land, Germany +DE Castroper Hellweg 109, 44805 B, Germany +MX Celaya Planta, GUANAJUATO, Mexico +US Cement, United States + Chandigarh +CN Chengdu, Sichuan, China; XiAn, Shaanxi, China +FR Cherbourg-en-Cotentin, Manche, France +US Chicago, Atlanta, US-Remote, Toronto +CL Chile + China - Hong Kong; Taiwan - Taipei +MX Ciudad de México, CDMX, Mexico +MX Colon, QUERETARO, Mexico +DK Copenhagen +DK Copenhagen Hub - Løvstræde, Denmark +DK Copenhagen Hub - Løvstræde; Billund +DK Copenhagen Hub - Østergade, Denmark +DK Copenhagen K, Denmark; Aarhus C, Denmark +DK Copenhagen O, Denmark +DK Copenhagen Office +DK Copenhagen S, Region Capital of Denmark, Denmark +DK Copenhagen V, Denmark +DK Copenhagen V, Denmark; Aarhus C, Denmark +DK Copenhagen or Aarhus + Copenhagen, Denmark; Tel Aviv, Israel +DK Copenhagen, Hovedstaden, Denmark +DK Copenhagen; Billund LEGO Campus + Copenhagen; Zurich; Singapore; Warsaw; Toronto +AR Corrientes, , Argentina + Corte Madera +NL Culemborg, Netherlands + Curitiba +PE Cusco, Cusco, Peru +DE DE Store Hamburg Spitalerstraße, Germany +US Dallas, United States +AU Dandenong South, VICTORIA, Australia +GB Darlington, England, United Kingdom +AU Deakin ACT, Australia +EU Denmark; Vilnius, Lithuania +DK Dianalund +CN Dongguan, Guangdong, China +DE Dortmund, Germany +DE Dortmund, North Rhineland Westphalia, DE +1 more… +IE Dublin, CA (ZICC) +EU Dublin, Ireland; London +GB Dublin, United Kingdom + Dunkerque + Durban +EU EMEA; Germany; London, United Kingdom; Paris, France; Remote - Netherlands +DE Ergolding, BY, Germany +DK Esbjerg, Region South Denmark, Denmark +DE Eschborn, HE, Germany +DE Eschenburg, HE, Germany + Ezhou +IT Fagagna, , Italy +GB Farnborough, Hampshire, United Kingdom +DK Farum +DK Finland, København, Litauen, Polen, Sverige +EU Finland; France; Germany; Netherlands; Remote - Europe; Sweden; United Kingdom +EU Finland; Remote - Denmark; Stockholm, Sweden + Finland; United Kingdom; Korea, Republic of; Denmark; France; India; Poland; United States + Finland; United Kingdom; United States; Belgium +DE Fins Only-DE-Munich-MSO +EU Fleet Place Office; Amsterdam Office; Berlin Office + France; India; Finland +DE Frankfurt; Berlin; Munich +DE Frankfurt; Munich +DK Fredericia or Ballerup +DK Fredericia, REGION_DNK 1852 South Denmark, Denmark +DK Frederiksberg +DK Frederikssund og mulighed for hjemmearbejde +DK Fårevejle +GB GB-London +US Garden City, TX, United States +EU Geneva; London +DE Gerlingen, Baden-Württemberg, Germany +DE Germany Scoober Office +EU Germany, Remote; The Netherlands, Remote + Germany; Portugal; Canada; United States +DK Glostrup +DK Glostrup, Denmark +PL Goleniów, Województwo zachodniopomorskie, Poland +US Greenville, South Carolina, United States, South Carolina, United States of America +PL Grodzisk Mazowiecki (mazowieckie), Poland +SE Göteborg, Västra Götaland, Sweden; Lund, Skåne, Sweden +AT Haag, Austria +DE Hannover, NDS, Germany +VN Hanoi, , Vietnam +DK Headquarters, Denmark +US Heber, AZ, United States +SE Helsingborg, Sweden +FI Helsinki Metropolitan Area +FI Helsinki, Finland +EU Helsinki, Finland; Stockholm, Sweden; Tallinn, Estonia +DK Herlev og mulighed for hjemmearbejde +DE Herzogenrath, North Rhine Westphalia, Germany +DE Hildesheim, NDS, Germany +DK Hillerød +DK Hillerød, Capital Region of Denmark, Denmark +DE Hoher Hagen 3, 58513 Lüdensche, Germany +DK Holbæk +DE Homburg, , Germany +DE Homburg/Saar, SL, Germany +EU Home Based - Americas; Home Based - APAC; Home based - EMEA; Home based - Worldwide +GB Home Based - Americas; Home based - EMEA; Office Based - London, UK +MX Home-based, Mexico +DK Hvalsø +DK Hvidovre + Hwaseong-si + Hybrid +GB Hybrid - London, England +IN Hyderabad; Noida +SE Härryda, Sweden +SE Hässleholm +SE Höör +JP Ibaraki, Osaka, Japan +US Illinois, USA, Remote; Minnesota, USA, Remote +DE Immenstadt im Allgäu, BY, Germany + Indija +IT Induno Olona, Italy + Israel; Remote - Europe; United Kingdom +DE Jacob-A.-Morand-Str. 18, 07552, Germany +JP Japan, Tokyo + Jena +CZ Jihlava 1, Kraj Vysočina, Czechia +NL Joure, Netherlands +DK Kalundborg, Region Zealand, Denmark +DK Kastrup +SE Katrineholm, Sweden +DE Kelsterbach, HE, Germany + Kempten im Allgäu +IL Kfar Saba, Israel + Konstanz +SE Kristianstad, Sweden +MY Kuala Lumpur – Co-labs, Kuala Lumpur – Co-labs, Malaysia +MY Kuala Lumpur, Malaysia +SE Kungsängen, Sweden +BE Kuurne, Belgium +DK København +DK København K +DK København K eller Odense C +DK København K og mulighed for hjemmearbejde +DK København NV +DK København S +DK København SV eller Aalborg og mulighed for hjemmearbejde +DK København SV, Odense C +DK København V eller Korsør +DK København V og mulighed for hjemmearbejde +DK København Ø og Frederiksberg C samt mulighed for hjemmearbejde +DK København Ø, Denmark +EU København, Capital RegionDenmark, Denmark; Moss, Østfold, Norway; Oslo, Oslo, Norway +DK København, Denmark +DK København, Denmark; Aarhus, Denmark; Aalborg, Denmark; Kolding, Denmark +DK København, Denmark; Aarhus, Denmark; Kolding, Denmark; Aalborg, Denmark +EU København, Denmark; Stockholm, Sweden +DK København, København SV +DK København, København SV, Odense, Odense C +DK København, København SV, Viby J, Aarhus +DK København, Roskilde +DK Køge +DK Køge, Denmark +ES La Muela (Zaragoza), Z, Spain + La Roche-sur-Yon +US Lake Zurich, Illinois, United States +SE Landskrona +EU Lausanne; London +DK Lem St, Denmark +DK Lem St, Region Central Jutland, Denmark +NL Lemelerveld, Netherlands +MX León, GUA, Mexico +IT Liguria, Italy +DE Lohr am Main, BY, Germany +SE Lomma + London, Canada; Remote, United States; Oak Brook, United States +GB London, London, United Kingdom + London, Paris, Toronto, New York +EU London, United Kingdom; Oslo, Norway + London, United Kingdom; Remote, United States +GB London; Billund Project House +EU London; Copenhagen +GB London; Remote (UK) +EU London; Warsaw; Copenhagen +EU London; Zurich +SE Luleå, Sweden +SE Lund, Skåne län, Sweden +SE Lund, Skåne, Sweden +US Madison, Wisconsin, United States of America +SE Malmo, Sweden +SE Malmö +SE Malmö, Skåne County, Sweden +SE Malmö, Skåne Län, Sweden +SE Malmö, Sweden +DK Maribo +DE Meschede, Nordrhein-Westfalen, Germany + Metz +US Miami +US Miami, Florida, United States +DK Middelfart, Denmark +AU Midland, Australia +IT Milan Office +EU Milan, Italy; Paris, Paris, France +US Minneapolis, Minnesota, United States, Minnesota, United States of America +US Minneola, KS, United States + Monterrey +MX Monterrey, Mexico +US Mountain View, CA, USA; San Francisco, CA, USA +US Mountain View, California +US Mountain View, California; Plano, Texas; San Francisco, California +DE Munich +EU Munich - Berlin - London - Paris +EU Munich - Berlin - London - Paris - Warsaw - Tallinn +EU Munich - Berlin - Paris +DE Munich, Bavaria, Germany +DK Munkebo, Denmark +LU Munsbach, Luxembourg + Mátészalka + Möhnesee +DE München +DE München (DE) +DE München, BAYERN, Germany +DE München, BY, Germany +DE München, Germany +US NYC +US NYC, SF, Chi, Remote + Nagpur +DK Nakskov, Denmark +DK Nakskov, Region Zealand, Denmark + Nanjing +JP Nasushiobara, Tochigi, Japan +GB New Britain + New York City, Toronto, Chicago, or Remote +US New York, San Francisco, Seattle, Atlanta, Chicago + New York; Hong Kong; London; Singapore; Sydney +RS Niš, Serbia +DK Nordhavn eller Aalborg og mulighed for hjemmearbejde +DK Nordhavn eller Viby J +DK Nordhavn or Åbyhøj +NO Norway +AU Nundah, Australia +SE Nybro, Sweden +DK Nykøbing F +DK Nykøbing F, Denmark +DK Næstved +DK Nørre Asmindrup +DE Nürnberg +CA ON, Toronto, Canada +DE Oberkochen; München +PL Oborniki, Poland +DK Odense C, Denmark +DK Odense M, Denmark +DK Odense, Denmark +DK Odense, REGION_DNK 1852 South Denmark, Denmark +GB Office Based - London, UK +TW Office Based - Taipei, Taiwan +US Oklahoma, United States +CA Ontario - Remote +US Ontario, CA, United States of America +US Orange County , California, United States +SE Oskarshamn, Sweden +EU Paris, France; Copenhagen, Denmark; Amsterdam, Netherlands; Berlin, Germany +FR Paris, France; Tassin-la-Demi-Lune, France; Bordeaux, France +EU Paris;Geneva +US Paw Paw, IL, United States +FR Perigny, France +FR Perpignan, Pyrénées-Orientales, France +SE Perstorp +US Phillipsburg, New Jersey, United States +DE Pinkertweg 16, 22113 Hamburg, Germany +MX Planta Celaya, , Mexico +EU Poland; Hungary +PT Porto, Portugal +EU Porto, Portugal; Madrid, Spain; Berlin, Germany +CZ Prague +EC Quito, , Ecuador +DK Randers SV, København V or Krakow +GB Redhill (GB) +DK Region Hovedstaden +DK Region Sjælland +IL Rehovot, Israel +AU Remote - Australia +CA Remote - CAN British Columbia + Remote - Portugal; United States +US Remote - USA WA, United States of America +US Remote - United States +DK Remote OR Copenhagen +US Remote US MD, United States of America + Remote job + Remote, AMER + Remote, Canada; Remote, Poland; Remote, United Kingdom; Remote, United States + Remote, Canada; Remote, United Kingdom; Remote, US +EU Remote, Germany; Dublin, Ireland; Lisbon, Portugal +US Remote, US, Georgia +US Remote, Washington, USA, United States of America +BG Remote- Bulgaria + Remote-NORAM +US Remote-US +RS Remote; Serbia +DE Remscheid +CA Retail Store Chinook Ctr, Canada +US Retail Store Ontario Mills, United States of America +HR Rijeka, Croatia +DK Ringkøbing, Denmark +DK Ringsted +DK Ringsted, Ringsted +BR Rio de Janeiro, Brasil +DE Rosenheim +DK Roskilde +DK Roskilde og mulighed for hjemmearbejde +DK Roskilde, Denmark +PL Rosówek, Poland + Roßdorf +AU Ryde, Australia +DE Röntgenstr. 33, 32107 Bad Salz, Germany +DK Rønne or Køge + Saint-Etienne + Saint-Malo +US San Bernardino, California, United States, California, United States of America +US San Francisco, CA; Washington, DC +US San Francisco, Seattle, New York, Chicago, Remote in the US +CR San Jose, Costa Rica +HN San Pedro Sula, , Honduras +MT San Ġwann, Malta +DE Schwäbisch Gmünd, BW, Germany +US Seattle, Washington, United States +US Seattle, Washington, United States, Washington, United States of America +US Seymour, United States +CN Shanghai, Shanghai, China; Suzhou, Jiangsu, China +PL Siedlc, Poland +SE Sigtuna, Sweden +SG Singapore - Singapore +SG Singapore, SG +DK Slagelse +SE Sollentuna, Sweden +SE Solna +BR Sorocaba, SP, Brazil +DK Sorø +GB Southampton, England, United Kingdom +SE Staffanstorp +US Stamford, Connecticut, United States, Connecticut, United States of America +SE Stockholm HQ +SE Stockholm, Stockholm County, Sweden +EU Stockholm, Stockholm County, Sweden; London, London, United Kingdom; Barcelona, Spain +SE Stockholm, Stockholm County, Sweden; Malmö, Sweden +EU Stockholm, Sweden; Berlin, Germany +EU Stockholm, Sweden; Jorvas, Kirkkonummi, Finland +EU Stockholm; London; Milan; Warsaw +CZ Strakonice, Czechia +SE Svedala +SE Sweden +SE Sweden - Linköping +SE Sweden - Lund +GB Swindon, United Kingdom +AU Sydney Or Melbourne +US Syracuse, New York, United States, New York, United States of America +PT São João, Aveiro, Portugal +SE Södertälje, Sweden +DK Taastrup +DK Taastrup og mulighed for hjemmearbejde +TW Taichung, Taichung City, Taiwan +TW Taipei, Taiwan, Taiwan + Taipei; Geneva +CN Taiyuan, Shanxi, China +EE Tallinn, Estonia +GE Tbilisi, Georgia +MX Tecnoparque, Mexico City, Mexico; Santa Fe, Mexico City, Mexico; Mexico City, Mexico +MX Tepotzotlan, Estado de México, Mexico +US The Woodlands, United States of America +RO Timisoara, Romania +GB Tiverton, United Kingdom +SE Tomelilla +MX Tonalá, Mexico + Tonsley SA + Toronto, New York, San Francisco, Atlanta, Seattle +PL Toruń, PM, Poland +PL Toruń, Poland +SE Trelleborg + Troisdorf +NO Trondheim, Norway + Tunis +DE Tübingen (DE) +US US-IL-Chicago-MSO + USA (Remote), Spain (Remote), UK (Remote) +US USA - IL - Chicago, United States of America +US USA - UT - Remote, United States of America +GB United Kingdom Remote +EU United Kingdom; Paris, Ile-de-France, France + United States; Stockholm, Sweden; Melbourne, Victoria, Australia + United States; Sweden +CN Urumqi, Xinjiang, China +FR Valbonne, Provence-Alpes-Côte d'Azur, France +DK Valby, Aalborg +DK Vallensbæk Strand +CA Vancouver + Vannes +FI Vantaa, Uusimaa, Finland +SE Varberg, Sweden +SE Vellinge +US Vernon Hills, IL, United States +DK Vi søger i hele landet +US Victorville, California, United States +DK Videbæk, Denmark +DK Videbæk, Region Central Jutland, Denmark +EU Vilnius, Lithuania; Copenhagen K, Denmark +EU Vilnius, Lithuania; Copenhagen V, Denmark +DK Værløse +EU Warsaw, Poland; Vilnius, Lithuania +PL Warszawa +AU Wensleydale, VIC, Australia +DE Wiesbaden, HE, Germany +CA Windsor, Canada +US Windsor, United States +GB Worksop, UK + Wuhan +CN XiAn, Shaanxi, China +SE Ystad +CN Zhuhai, Guangdong, China +SE Älmhult, Sweden +SE Ängelholm +SE Ängelholm, Sweden +SE Åby, Sweden +DK Åbyhøj eller Ballerup +SE Örebro, Sweden +SE Östra Göinge +CN 东莞, , China +""" + +LINES = [tuple(reversed(line.split("\t", 1))) for line in REAL.strip("\n").split("\n")] + +#: The tokens of every kind that are known. (`country:ZZ` and the rest that name +#: nothing are `known` false and match nothing, which the old LIKE did not say.) +ORACLE_TOKENS = [ + "country:DK", "country:SE", "country:DE", "country:GB", "country:US", "country:EU", + "country:NO", "country:CH", + "region:skane", "region:hovedstaden", "region:zealand", "region:oresund", + "region:jutland", "region:funen", + "place:copenhagen", "place:kobenhavn", "place:malmo", "place:lund", "place:munich", + "place:zurich", "place:london", "place:remote", "place:aarhus", "place:kgs lyngby", + "place:the hague", "place:scania", "place:capital region of denmark", + "place:denmark", "place:germany", "place:skane", "Skåne", "Berlin", +] + + +def like_words(column: str, phrases) -> tuple[str, list]: + """The clause `match_sql` was before it called a function: each phrase a + `LIKE` over the padded column. Slow, and the definition `matches` must keep.""" + phrases = [p for p in dict.fromkeys(phrases) if p] + clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'" + return " OR ".join([clause] * len(phrases)) or "0", [f"% {p} %" for p in phrases] + + +def like_country(code: str) -> tuple[str, list]: + names, params = like_words("place_fold", places._COUNTRY_MENTIONS.get(code, ())) + if not params: + return "country = ?", [code] + return f"country = ? OR (country IN ('', 'EU') AND ({names}))", [code, *params] + + +def like_sql(token: str) -> tuple[str, list]: + tok = parse(token) + if tok.kind == "country": + sql, params = like_country(tok.value) + elif tok.kind == "region": + sql, params = like_words("place_fold", places._PHRASES[tok.value]) + else: + region = places._REGION_BY_NAME.get(tok.value, "") + sql, params = like_words("place_fold", [*places._expand(tok.value), + *places._PHRASES.get(region, ())]) + if tok.value in places._COUNTRY_BY_NAME: + more, extra = like_country(places._COUNTRY_BY_NAME[tok.value]) + sql, params = f"{sql} OR {more}", [*params, *extra] + return f"({sql})", params + + +@pytest.fixture(scope="module") +def real() -> sqlite3.Connection: + db = connect() + db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") + db.executemany("INSERT INTO roles VALUES (?, ?, ?)", + [(place, fold(place), country) for place, country in LINES]) + return db + + +def test_there_are_enough_real_lines() -> None: + assert len(set(LINES)) >= 300 + assert {"DK", "SE", "DE", "EU", ""} <= {country for _, country in LINES} + + +@pytest.mark.parametrize("token", ORACLE_TOKENS) +def test_matches_agrees_with_the_like_it_replaces(real: sqlite3.Connection, token: str) -> None: + sql, params = like_sql(token) + by_like = {row[0] for row in real.execute( + f"SELECT rowid FROM roles WHERE {sql}", params)} + by_function = {row[0] for row in real.execute( + "SELECT rowid, place_fold, country FROM roles") if matches(row[1], row[2], token)} + assert by_function == by_like, token + sql, params = match_sql(token) + assert {row[0] for row in real.execute( + f"SELECT rowid FROM roles WHERE {sql}", params)} == by_like + + +@pytest.mark.parametrize("kind, tokens", [ + ("country", [t for t in ORACLE_TOKENS if t.startswith("country:")]), + ("region", [t for t in ORACLE_TOKENS if t.startswith("region:")]), + ("place", [t for t in ORACLE_TOKENS if t.startswith("place:")]), +]) +def test_each_kind_finds_something_in_the_real_lines(real: sqlite3.Connection, kind: str, + tokens: list[str]) -> None: + """Agreement on nothing would prove little: most tokens hit, some lines do not.""" + hits = [sum(matches(fold(place), country, t) for place, country in LINES) for t in tokens] + assert sum(1 for n in hits if n) >= len(tokens) * 0.75, (kind, hits) + assert all(n < len(LINES) for n in hits) + + +def test_a_region_over_thousands_of_lines_is_fast() -> None: + """The reason `matches` is a function: 160 `LIKE` clauses took 400 ms per query. + The bound is generous, a machine under load is not the test's business.""" + import time + + db = connect() + db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)") + rows = [(f"{place} {i}", fold(f"{place} {i}"), country) + for i in range(10) for place, country in LINES] + db.executemany("INSERT INTO roles VALUES (?, ?, ?)", rows) + assert len(rows) >= 5000 + for token in ("region:oresund", "region:hovedstaden", "region:skane", "country:DK"): + sql, params = match_sql(token) + start = time.perf_counter() + n = db.execute(f"SELECT count(*) FROM roles WHERE {sql}", params).fetchone()[0] + assert time.perf_counter() - start < 0.5, token + assert n > 0 + + +# --------------------------------------------- what is suggested, and on what + + +def test_oresund_is_not_suggested_for_the_names_of_the_regions_it_holds() -> None: + for q in ("ska", "skane", "den", "capital", "hoved", "scania"): + assert "region:oresund" not in tokens(q), q + for q in ("ore", "øre", "oresund", "öresund"): + assert "region:oresund" in tokens(q), q + + +def test_a_region_is_suggested_on_its_own_names() -> None: + assert tokens("ska")[:1] == ["region:skane"] + assert "region:hovedstaden" in tokens("den") + assert "region:zealand" in tokens("den") or "region:zealand" in tokens("zea") + + +def test_a_line_that_holds_a_town_counts_under_it() -> None: + munich = [s for s in suggest("mun", PLACES) if s.kind == "place"] + assert [s.token for s in munich] == ["place:munich"] + # Munich 40, München 10, Office 5, (DE) 3, Munich - Berlin 2, Garching (Munich) 4. + assert munich[0].count == 40 + 10 + 5 + 3 + 2 + 4 + assert munich[0].label == "Munich" + + +def test_a_line_that_holds_two_towns_counts_under_both() -> None: + berlin = [s for s in suggest("ber", PLACES) if s.kind == "place"] + assert [(s.label, s.count) for s in berlin] == [("Berlin", 100 + 2)] + + +def test_copenhagen_with_an_office_or_a_hub_is_copenhagen() -> None: + towns = [s for s in suggest("cope", PLACES) if s.kind == "place"] + assert [s.token for s in towns] == ["place:kobenhavn"] + assert towns[0].count == 69 + 35 + 34 + 1 + 56 + 1 + 16 + 14 + 2 + for text in ("Copenhagen Office", "Hub", "MSO"): + assert all(text not in s.label for s in towns) + + +def test_a_town_the_tables_do_not_know_stays_as_it_is() -> None: + labels = {s.label for q in "abcdefghijklmnopqrstuvwxyz" + for s in suggest(q, PLACES, limit=999) if s.kind == "place"} + assert {"Albertslund", "Billund", "Auckland"} <= labels + assert not {"Copenhagen Office", "Munich Office", "Munich - Berlin", "Munich (DE)"} & labels + + +def test_a_line_that_is_only_a_known_town_in_a_word_is_not_a_hit_for_another() -> None: + """`Lund` is not in `Billund`: a line holds a town as whole words or not at all.""" + assert [s.label for s in suggest("lund", PLACES) if s.kind == "place"] == ["Lund"] + + +def test_a_bar_separates_places_like_a_semicolon() -> None: + rows = [("Denmark | Remote", "DK", 1), ("Lund | Malmö", "SE", 2)] + assert [s.label for s in suggest("den", rows) if s.kind == "place"] == [] + assert [(s.label, s.count) for s in suggest("mal", rows) if s.kind == "place"] \ + == [("Malmö", 2)]