"""Places as a filter: what a board writes in `place`, matched by whole words. A location line is free text: "København K", "Kgs. Lyngby", "Aarhus N, Region Central Jutland, Denmark". A substring match finds `Lund` in `Billund` and `Denmark` in nothing that only names a town. Here a place is folded (casefolded, marks stripped, `ø` to `o`, `æ` to `ae`, `å` to `a`, punctuation to spaces) and matched as whole words, so `Lund` is `lund` and no other word. A filter is one of three tokens, and several tokens are joined by OR: country:DK the role's country, or a line that names it while `country` is empty or `EU` (a board that lists several offices) region:skane any town or name of a region in `REGIONS` place:malmo the town's fold and its aliases, so `copenhagen`, `kobenhavn` and `kopenhamn` find each other `match_sql` turns a token into SQL over `roles.place_fold` and `roles.country`; `suggest` turns typed text into tokens for the autocomplete; `label` reads a chip's text back from the address. No database access here, the caller hands over the places it holds. """ from __future__ import annotations import re from collections import Counter from collections.abc import Iterable from dataclasses import dataclass from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold __all__ = ["REGIONS", "Region", "Suggestion", "Token", "fold", "label", "match_sql", "parse", "suggest"] def fold(text: str) -> str: """`Skåne län` -> `skane lan`, `København K` -> `kobenhavn k`, `Kgs. Lyngby` -> `kgs lyngby`. What `roles.place_fold` holds. `å` needs no rule of its own: NFKD splits it into `a` and a ring, and the ring is a mark.""" return _place_fold(text) # ------------------------------------------------------------------ countries def _countries() -> dict[str, tuple[str, ...]]: """Every code to the names `_COUNTRY_NAMES` gives it, English name first.""" found: dict[str, tuple[str, ...]] = {} for line in _COUNTRY_NAMES.split("\n"): if line: code, rest = line.split(" ", 1) found[code] = tuple(name.strip() for name in rest.split("|")) return found _COUNTRY_LABELS = _countries() #: A code to the folded names a country goes by. _COUNTRY_FOLDS: dict[str, tuple[str, ...]] = { code: tuple(dict.fromkeys(filter(None, map(fold, names)))) for code, names in _COUNTRY_LABELS.items()} #: What a line has to say for a role of that country whose `country` column is #: empty or `EU`: a name of the country, or a city `country_of` knows in it. A #: board that lists offices ("Copenhagen; London; Warsaw") names cities only. _COUNTRY_MENTIONS: dict[str, tuple[str, ...]] = { code: tuple(dict.fromkeys((*names, *(fold(c) for c, cc in _CITIES.items() if cc == code)))) for code, names in _COUNTRY_FOLDS.items()} #: A folded country name to its code. _COUNTRY_BY_NAME: dict[str, str] = { name: code for code, names in _COUNTRY_FOLDS.items() for name in names} #: Words that are not a town: a country, a macro region, how the role is worked. _NOT_A_TOWN = frozenset(_COUNTRY_BY_NAME) | frozenset(map(fold, _REGIONS)) \ | {"remote", "hybrid"} # -------------------------------------------------------------------- aliases #: Towns boards write in more than one language, first spelling shown. Each #: group is symmetric: any spelling finds the others. The Nordics and the DACH #: countries in full for their large cities, and the European cities that #: appear both ways in boards. _ALIAS_GROUPS: tuple[tuple[str, ...], ...] = ( # Denmark ("Copenhagen", "København", "Kobenhavn", "Köpenhamn", "Kopenhamn", "Kopenhagen", "Copenhague", "Kopenhaga"), ("Aarhus", "Århus"), ("Aalborg", "Ålborg"), ("Kgs. Lyngby", "Kongens Lyngby", "Lyngby"), ("Taastrup", "Tåstrup"), ("Elsinore", "Helsingør"), # Sweden ("Gothenburg", "Göteborg", "Goteborg"), ("Stockholm",), ("Malmö", "Malmo"), ("Helsingborg", "Hälsingborg"), ("Västerås", "Vasteras"), ("Linköping", "Linkoping"), ("Örebro", "Orebro"), ("Norrköping", "Norrkoping"), ("Jönköping", "Jonkoping"), # Norway, Finland, Iceland ("Oslo",), ("Bergen",), ("Trondheim", "Trondhjem"), ("Helsinki", "Helsingfors"), ("Espoo", "Esbo"), ("Vantaa", "Vanda"), ("Reykjavik", "Reykjavík"), # Germany ("Munich", "München", "Muenchen", "Munchen"), ("Cologne", "Köln", "Koeln", "Koln"), ("Nuremberg", "Nürnberg", "Nuernberg", "Nurnberg"), ("Düsseldorf", "Duesseldorf", "Dusseldorf"), ("Hanover", "Hannover"), ("Mainz", "Mayence"), ("Frankfurt", "Frankfurt am Main"), ("Göttingen", "Goettingen", "Gottingen"), ("Würzburg", "Wuerzburg", "Wurzburg"), ("Tübingen", "Tuebingen", "Tubingen"), ("Lübeck", "Luebeck", "Lubeck"), ("Münster", "Muenster", "Munster"), # Austria ("Vienna", "Wien", "Vienne"), ("Salzburg",), ("Innsbruck",), # Switzerland ("Zurich", "Zürich", "Zuerich"), ("Geneva", "Genève", "Geneve", "Genf", "Ginevra"), ("Basel", "Basle", "Bâle"), ("Bern", "Berne"), ("Lucerne", "Luzern"), ("St. Gallen", "Sankt Gallen", "Saint Gall"), # The rest of Europe ("Prague", "Praha", "Prag"), ("Warsaw", "Warszawa", "Warschau"), ("Krakow", "Kraków", "Cracow"), ("Wroclaw", "Wrocław", "Breslau"), ("Gdansk", "Gdańsk", "Danzig"), ("Poznan", "Poznań", "Posen"), ("Lisbon", "Lisboa", "Lissabon"), ("Milan", "Milano", "Mailand"), ("Rome", "Roma", "Rom"), ("Turin", "Torino"), ("Florence", "Firenze"), ("Naples", "Napoli", "Neapel"), ("Venice", "Venezia", "Venedig"), ("Genoa", "Genova"), ("Brussels", "Bruxelles", "Brussel", "Brüssel"), ("Antwerp", "Antwerpen", "Anvers"), ("Ghent", "Gent", "Gand"), ("Bruges", "Brugge"), ("Liège", "Liege", "Luik", "Lüttich"), ("The Hague", "Den Haag", "s-Gravenhage"), ("Luxembourg", "Luxemburg"), ("Seville", "Sevilla"), ("Athens", "Athina", "Athen"), ("Bucharest", "București", "Bucuresti"), ("Belgrade", "Beograd"), ("Bratislava", "Pressburg"), ("Kyiv", "Kiev"), ("Cluj-Napoca", "Cluj"), ("Timișoara", "Timisoara"), ("Marseille", "Marseilles"), ("Bolzano", "Bozen"), ) def _variants(text: str) -> tuple[str, ...]: """A fold and its `aa` spelling: Danish writes `å` as `aa` (`Aarhus`, `Taastrup`) and `Århus` and `Tåstrup` fold to `arhus` and `tastrup`.""" return tuple(dict.fromkeys((text, text.replace("aa", "a")))) def _aliases() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]: """Every folded spelling to its whole group (all folded), and to its shown spelling. The group starts with the first spelling listed.""" groups: dict[str, tuple[str, ...]] = {} shown: dict[str, str] = {} for group in _ALIAS_GROUPS: members = tuple(dict.fromkeys(v for name in group for v in _variants(fold(name)))) for name in group: shown.setdefault(fold(name), name) for member in members: groups[member] = members return groups, shown _GROUPS, _SHOWN = _aliases() def _expand(text: str) -> tuple[str, ...]: """A folded town and everything it is also written as.""" return _GROUPS.get(text) or _variants(text) # -------------------------------------------------------------------- regions @dataclass(frozen=True) class Region: """A region below a country, or across two. `names` are what a board calls the region itself, `towns` the places in it; a line is in the region when it has one of either as whole words. Spellings are shown with their marks and folded when used.""" key: str label: str names: tuple[str, ...] towns: tuple[str, ...] country: str = "" def _split(text: str) -> tuple[str, ...]: return tuple(part.strip() for part in text.split(",") if part.strip()) _SKANE = Region( "skane", "Skåne", _split("Skåne, Skåne län, Skåne County, Region Skåne, Skåne Region, Scania, Skaane"), _split("""Malmö, Lund, Helsingborg, Kristianstad, Landskrona, Trelleborg, Ängelholm, Hässleholm, Ystad, Eslöv, Höganäs, Staffanstorp, Lomma, Bjuv, Burlöv, Kävlinge, Svedala, Vellinge, Simrishamn, Tomelilla, Sjöbo, Svalöv, Klippan, Åstorp, Bromölla, Osby, Perstorp, Örkelljunga, Östra Göinge, Hörby, Höör, Båstad, Skurup, Arlöv, Limhamn, Dalby, Åkarp, Bjärred, Höllviken, Ljungbyhed, Löddeköpinge, Furulund, Genarp, Veberöd, Skanör, Falsterbo, Mölle, Arild"""), "SE") _HOVEDSTADEN = Region( "hovedstaden", "Capital Region of Denmark", _split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"), _split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup, Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte, Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre, Høje-Taastrup, Hørsholm, Ishøj, Lyngby-Taarbæk, Rudersdal, Rødovre, Tårnby, Vallensbæk, Kgs. Lyngby, Kongens Lyngby, Lyngby, Virum, Holte, Nærum, Birkerød, Vedbæk, Skodsborg, Klampenborg, Charlottenlund, Hellerup, Ordrup, Bagsværd, Søborg, Mørkhøj, Måløv, Skovlunde, Smørum, Ølstykke, Stenløse, Værløse, Farum, Slangerup, Hedehusene, Taastrup, Kastrup, Amager, Valby, Vanløse, Brønshøj, Nørrebro, Østerbro, Vesterbro, Nordhavn, Ørestad, Sydhavn, Islands Brygge, Refshaleøen, Humlebæk, Kokkedal, Espergærde, Snekkersten, Rungsted, Rungsted Kyst, Lillerød, Frederiksværk, Hundested, Helsinge, Gilleleje, Græsted, Vallensbæk Strand, Skævinge, Rønne, Nexø, Allinge, Svaneke, Aakirkeby, Hasle"""), "DK") # Region Sjælland, not the island: Copenhagen is on Zealand too, but a board that # writes "Region Zealand" means the region and the Capital Region has its own key. # The bare word is left out of `names` since "New Zealand" would match it. _ZEALAND = Region( "zealand", "Zealand", _split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"), _split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved, Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg, Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing, Haslev, Korsør, Skælskør, Præstø, Stege, Fakse, Rødby, Rødbyhavn, Jyllinge, Viby Sjælland, Solrød Strand, Fuglebjerg, Gørlev, Dianalund, Jyderup, Hvalsø, Tølløse, Hørve, Fårevejle, Bjæverskov, Nørre Asmindrup"""), "DK") _JUTLAND = Region( "jutland", "Jutland", _split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland, North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland, East Jutland, West Jutland, Vestjylland, Østjylland"""), _split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning, Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov, Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå, Hjørring, Frederikshavn, Skagen, Thisted, Struer, Ikast, Brande, Varde, Ribe, Tønder, Vojens, Hadsten, Hobro, Skanderborg, Odder, Hedensted, Vejen, Ebeltoft, Lemvig, Rønde, Egå, Lystrup, Hinnerup, Tilst, Hammel, Galten, Sabro, Harlev, Løsning, Højbjerg, Juelsminde, Nørresundby, Brønderslev, Aars, Nibe, Sæby, Løgstør, Tarm, Skjern, Padborg, Rødekro, Toftlund, Bramming, Kruså, Brabrand, Ry, Gjern"""), "DK") _FUNEN = Region( "funen", "Funen", _split("Funen, Fyn, Region Fyn"), _split("""Odense, Svendborg, Nyborg, Faaborg, Fåborg, Middelfart, Assens, Kerteminde, Bogense, Langeskov, Ringe, Otterup, Munkebo, Vissenbjerg, Årslev, Tommerup, Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""), "DK") # The Øresund region is the two capital regions either side of the sound. _ORESUND = Region( "oresund", "Øresund", _split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen") + _HOVEDSTADEN.names + _SKANE.names, _HOVEDSTADEN.towns + _SKANE.towns) REGIONS: dict[str, Region] = { r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)} def _phrases(region: Region) -> tuple[str, ...]: """Every folded phrase that puts a line in the region, with the other spellings of its towns.""" found: dict[str, None] = {} for text in (*region.names, *region.towns): for form in _expand(fold(text)): found[form] = None return tuple(found) _PHRASES: dict[str, tuple[str, ...]] = {key: _phrases(r) for key, r in REGIONS.items()} #: A folded key, label or name to the region key; a name in two regions ("Capital #: Region of Denmark" is also in Øresund) belongs to the one that lists it first. _REGION_BY_NAME: dict[str, str] = {} for _key, _region in REGIONS.items(): for _name in (_region.key, _region.label, *_region.names): _REGION_BY_NAME.setdefault(fold(_name), _key) #: What a place is shown as when a chip is read from the address: every alias #: spelling, every region and every town of one, as written above. _TOWN_LABELS: dict[str, str] = dict(_SHOWN) for _region in REGIONS.values(): for _town in (_region.label, *_region.towns, *_region.names): _TOWN_LABELS.setdefault(fold(_town), _town) _NOT_A_TOWN |= frozenset(_REGION_BY_NAME) # --------------------------------------------------------------------- tokens @dataclass(frozen=True) class Token: """One place filter. `value` is a country code, a region key or a fold.""" kind: str value: str label: str def __str__(self) -> str: return f"{self.kind}:{self.value}" def _region_key(text: str) -> str: folded = fold(text) return _REGION_BY_NAME.get(folded, folded) def parse(token: str) -> Token: """`country:DK`, `region:skane` or `place:malmo`. Bare text, or a prefix that is none of these, is a place: `Skåne` is `place:skane`. A country by name (`country:Denmark`) and a region by label (`region:Skåne`) are read as well.""" kind, sep, value = (token or "").partition(":") kind = kind.strip().lower() if not sep or kind not in ("country", "region", "place"): kind, value = "place", token or "" value = value.strip() if kind == "country": code = value.upper() if code not in _COUNTRY_LABELS: code = _COUNTRY_BY_NAME.get(fold(value), code) return Token("country", code, _COUNTRY_LABELS.get(code, (code,))[0]) if kind == "region": key = _region_key(value) return Token("region", key, REGIONS[key].label if key in REGIONS else key) folded = fold(value) return Token("place", folded, _TOWN_LABELS.get(folded) or folded.title()) def label(token: str | Token) -> str: """The text of a chip read from the address: `country:DK` is `Denmark`, `region:skane` is `Skåne`, `place:malmo` is `Malmö`. A town the tables do not know is its fold in title case.""" return (token if isinstance(token, Token) else parse(token)).label # ------------------------------------------------------------------------ SQL _NEVER = ("0", []) def _escape(word: str) -> str: """A folded word holds no wildcard, but a caller may hand over raw text.""" return re.sub(r"([\\%_])", r"\\\1", word) def _words(column: str, phrases: Iterable[str]) -> tuple[str, list]: """Any of the phrases as whole words of the column. A multi-word phrase is a phrase: `kgs lyngby` needs both words, side by side.""" phrases = [p for p in dict.fromkeys(phrases) if p] if not phrases: return _NEVER clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'" return " OR ".join([clause] * len(phrases)), [f"% {_escape(p)} %" for p in phrases] def _country(code: str, place: str, country: str) -> tuple[str, list]: """The role's own country, or a line that names the country while `country` is empty or `EU`: a board listing several offices has no single country.""" names, params = _words(place, _COUNTRY_MENTIONS.get(code, ())) if not params: return f"{country} = ?", [code] return f"{country} = ? OR ({country} IN ('', 'EU') AND ({names}))", [code, *params] def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \ -> tuple[str, list]: """SQL over the two columns for one token, parenthesised, with `?` parameters. Several tokens are joined with OR by the caller. Bare text is a place. A `place:` value that is also a region or a country is matched as that too: `place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate written as a bare word means what the owner meant by it. A token that names nothing known (`region:atlantis`, an empty value) matches no role.""" tok = parse(token) if tok.kind == "country": sql, params = _country(tok.value, place, country) elif tok.kind == "region": sql, params = _words(place, _PHRASES.get(tok.value, ())) else: region = _REGION_BY_NAME.get(tok.value) sql, params = _words(place, [*_expand(tok.value), *_PHRASES.get(region, ())]) \ if tok.value else _NEVER if tok.value in _COUNTRY_BY_NAME: more, extra = _country(_COUNTRY_BY_NAME[tok.value], place, country) sql, params = f"{sql} OR {more}", [*params, *extra] return f"({sql})", params # ---------------------------------------------------------------- suggestions @dataclass(frozen=True) class Suggestion: """`token` is what the chip holds, `kind` is `country`, `region` or `place`, `count` the open roles it stands for.""" token: str label: str kind: str count: int def _prefixes(q: str, candidates: Iterable[str]) -> bool: """`q` starts a word of a candidate, or a run of words: `kgs ly` finds `kgs lyngby`, `hague` finds `the hague`.""" return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates) #: Where one town ends in a line: `;` between places, and `or` / `eller` / `og` #: between two towns in one ("København V eller Aalborg"). _PART = re.compile(r"\s*;\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE) #: Danish boards append "and the possibility of working from home". _AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE) #: A postal district after the town, "København K", "Aarhus C", "Odense SV". _DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$") #: A country or "Remote" ahead of the town, "Sweden - Lund", "Remote-New Zealand". _LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$") def _towns(place: str) -> list[str]: """The first comma part of each place of a line: `Aarhus N, Region Central Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal district is dropped, since `place:kobenhavn` finds `København K` anyway.""" found = [] for part in _PART.split(_AND_HOME.sub("", place or "")): head = part.split(",")[0].strip() if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN: head = lead[2] head = _DISTRICT.sub("", head) if fold(head) and fold(head) not in _NOT_A_TOWN: found.append(head) return found def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]: """Open roles per region, over folded lines. A line is cut into every run of words once, and each region asks whether it holds one of its phrases.""" sets = {key: set(_PHRASES[key]) for key in keys} longest = max((len(p.split()) for key in sets for p in sets[key]), default=1) counts: Counter[str] = Counter() for text, roles in places: words = text.split() runs = {" ".join(words[i:i + n]) for n in range(1, longest + 1) for i in range(len(words) - n + 1)} for key, phrases in sets.items(): if not phrases.isdisjoint(runs): counts[key] += roles return counts def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \ -> list[Suggestion]: """Tokens for the text typed so far. `places` is `(place, country, open roles)`. Countries first, then regions, then towns, each by open roles. A country is suggested when `q` starts a word of one of its names, with the roles that carry its code. A region likewise on its label and names, with the roles whose line is in it. A town is the first comma part of each `;` part of a line, grouped by fold and by alias (`Copenhagen` and `København` are one town, shown as its commonest spelling). Countries, regions and towns with no open role are left out: a chip that finds nothing helps nobody.""" text = fold(q) if not text: return [] rows = [(place or "", country or "", int(roles)) for place, country, roles in places] by_country: Counter[str] = Counter() for _, country, roles in rows: by_country[country] += roles countries = [Suggestion(f"country:{code}", names[0], "country", by_country[code]) for code, names in _COUNTRY_LABELS.items() if by_country[code] and _prefixes(text, _COUNTRY_FOLDS[code])] named = [key for key, r in REGIONS.items() if _prefixes(text, [fold(r.label), *map(fold, r.names)])] counts = _region_counts(named, [(fold(place), roles) for place, _, roles in rows]) regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key]) for key in named if counts[key]] spellings: dict[str, Counter[str]] = {} written: dict[str, set[str]] = {} for place, _, roles in rows: for town in _towns(place): folded = fold(town) group = _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1] spellings.setdefault(group, Counter())[town] += roles written.setdefault(group, set()).add(folded) towns = [] for group, seen in spellings.items(): if not _prefixes(text, [form for folded in written[group] for form in _expand(folded)]): continue shown = min(seen, key=lambda s: (-seen[s], s)) towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", sum(seen.values()))) countries.sort(key=lambda s: (-s.count, s.label)) regions.sort(key=lambda s: (-s.count, s.label)) towns.sort(key=lambda s: (-s.count, s.label)) return (countries + regions + towns)[:limit]