remote:country:DK is a remote role whose area overlaps Denmark: what lies in it, what it lies in, Anywhere. remote:anywhere is Anywhere alone and remote:any the viewing agent's scope. Anywhere is a continent outside the tree with no alias. Remote parts of a line are no town, and the suggestions offer the remote tokens after the countries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1250 lines
56 KiB
Python
1250 lines
56 KiB
Python
"""Places as a filter: what a board writes in `place`, matched by whole words.
|
|
|
|
A location line is free text: "København K", "Kgs. Lyngby", "Aarhus N, Region
|
|
Central Jutland, Denmark". A substring match finds `Lund` in `Billund` and
|
|
`Denmark` in nothing that only names a town. Here a place is folded (casefolded,
|
|
marks stripped, `ø` to `o`, `æ` to `ae`, `å` to `a`, punctuation to spaces) and
|
|
matched as whole words, so `Lund` is `lund` and no other word. A filter is one
|
|
of three tokens, and several tokens are joined by OR:
|
|
|
|
country:DK the role's country, or a line that names it while `country`
|
|
is empty or `EU` (a board that lists several offices)
|
|
region:skane any town or name of a region in `REGIONS`
|
|
place:malmo the town's fold and its aliases, so `copenhagen`,
|
|
`kobenhavn` and `kopenhamn` find each other
|
|
|
|
`matches` answers whether one role's folded place and country fit a token, in
|
|
Python: a region is over a hundred phrases, and as `LIKE` clauses that costs
|
|
several hundred milliseconds a query, as a set intersection on the words of a
|
|
line it costs a few. `register` gives a connection that answer as the SQL
|
|
function `place_match`, and `match_sql` returns the clause that calls it.
|
|
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
|
|
chip's text back from the address.
|
|
|
|
The filter and the gates no longer match words: a role's places are resolved
|
|
against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
|
|
and a token is the places under the place it names. `matches` stays the rule
|
|
for a `place:` token the gazetteer does not know. This module opens no
|
|
database; the caller hands over the places it holds, or its connection.
|
|
|
|
A fourth token finds remote roles by where their hire may live:
|
|
|
|
remote:country:DK a remote role whose area (`role_remote`, written by
|
|
`app.prep`) overlaps Denmark: Denmark or a place in
|
|
it, a place Denmark lies in (Europe, Øresund), or
|
|
Anywhere
|
|
remote:anywhere a remote role whose area is Anywhere, and no other
|
|
remote:any every remote role in the viewing agent's scope, its
|
|
`require` gates on country (`Store`, which knows the
|
|
agent, answers it)
|
|
|
|
Anywhere is a place of its own, outside the tree (`ANYWHERE`), so `under`
|
|
never reaches every role and the Place filter stays a filter. It has no
|
|
alias: only a remote area holds it, never a role's places.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import sqlite3
|
|
from collections import Counter
|
|
from collections.abc import Callable, Iterable, Sequence
|
|
from dataclasses import dataclass
|
|
from functools import lru_cache
|
|
|
|
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
|
|
country_of, remote_of)
|
|
|
|
__all__ = ["ANYWHERE", "ANYWHERE_WORDS", "KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place",
|
|
"Region", "Seed", "Suggestion", "Token", "fold", "known", "label", "match_sql",
|
|
"matches", "parse", "placeless", "register", "remote_part", "seed", "suggest",
|
|
"towns"]
|
|
|
|
|
|
def fold(text: str) -> str:
|
|
"""`Skåne län` -> `skane lan`, `København K` -> `kobenhavn k`, `Kgs. Lyngby`
|
|
-> `kgs lyngby`. What `roles.place_fold` holds. `å` needs no rule of its own:
|
|
NFKD splits it into `a` and a ring, and the ring is a mark."""
|
|
return _place_fold(text)
|
|
|
|
|
|
# ------------------------------------------------------------------ countries
|
|
|
|
|
|
def _countries() -> dict[str, tuple[str, ...]]:
|
|
"""Every code to the names `_COUNTRY_NAMES` gives it, English name first."""
|
|
found: dict[str, tuple[str, ...]] = {}
|
|
for line in _COUNTRY_NAMES.split("\n"):
|
|
if line:
|
|
code, rest = line.split(" ", 1)
|
|
found[code] = tuple(name.strip() for name in rest.split("|"))
|
|
return found
|
|
|
|
|
|
_COUNTRY_LABELS = _countries()
|
|
|
|
#: A code to the folded names a country goes by.
|
|
_COUNTRY_FOLDS: dict[str, tuple[str, ...]] = {
|
|
code: tuple(dict.fromkeys(filter(None, map(fold, names))))
|
|
for code, names in _COUNTRY_LABELS.items()}
|
|
|
|
#: What a line has to say for a role of that country whose `country` column is
|
|
#: empty or `EU`: a name of the country, or a city `country_of` knows in it. A
|
|
#: board that lists offices ("Copenhagen; London; Warsaw") names cities only.
|
|
_COUNTRY_MENTIONS: dict[str, tuple[str, ...]] = {
|
|
code: tuple(dict.fromkeys((*names, *(fold(c) for c, cc in _CITIES.items() if cc == code))))
|
|
for code, names in _COUNTRY_FOLDS.items()}
|
|
|
|
#: A folded country name to its code.
|
|
_COUNTRY_BY_NAME: dict[str, str] = {
|
|
name: code for code, names in _COUNTRY_FOLDS.items() for name in names}
|
|
|
|
#: Words that are not a town: a country, a macro region, how the role is worked.
|
|
_NOT_A_TOWN = frozenset(_COUNTRY_BY_NAME) | frozenset(map(fold, _REGIONS)) \
|
|
| {"remote", "hybrid"}
|
|
|
|
|
|
# -------------------------------------------------------------------- aliases
|
|
|
|
#: Towns boards write in more than one language, first spelling shown. Each
|
|
#: group is symmetric: any spelling finds the others. The Nordics and the DACH
|
|
#: countries in full for their large cities, and the European cities that
|
|
#: appear both ways in boards.
|
|
_ALIAS_GROUPS: tuple[tuple[str, ...], ...] = (
|
|
# Denmark
|
|
("Copenhagen", "København", "Kobenhavn", "Köpenhamn", "Kopenhamn", "Kopenhagen",
|
|
"Copenhague", "Kopenhaga"),
|
|
("Aarhus", "Århus"),
|
|
("Aalborg", "Ålborg"),
|
|
("Kgs. Lyngby", "Kongens Lyngby", "Lyngby"),
|
|
("Taastrup", "Tåstrup"),
|
|
("Elsinore", "Helsingør"),
|
|
# Sweden
|
|
("Gothenburg", "Göteborg", "Goteborg"),
|
|
("Stockholm",),
|
|
("Malmö", "Malmo"),
|
|
("Helsingborg", "Hälsingborg"),
|
|
("Västerås", "Vasteras"),
|
|
("Linköping", "Linkoping"),
|
|
("Örebro", "Orebro"),
|
|
("Norrköping", "Norrkoping"),
|
|
("Jönköping", "Jonkoping"),
|
|
# Norway, Finland, Iceland
|
|
("Oslo",),
|
|
("Bergen",),
|
|
("Trondheim", "Trondhjem"),
|
|
("Helsinki", "Helsingfors"),
|
|
("Espoo", "Esbo"),
|
|
("Vantaa", "Vanda"),
|
|
("Reykjavik", "Reykjavík"),
|
|
# Germany
|
|
("Munich", "München", "Muenchen", "Munchen"),
|
|
("Cologne", "Köln", "Koeln", "Koln"),
|
|
("Nuremberg", "Nürnberg", "Nuernberg", "Nurnberg"),
|
|
("Düsseldorf", "Duesseldorf", "Dusseldorf"),
|
|
("Hanover", "Hannover"),
|
|
("Mainz", "Mayence"),
|
|
("Frankfurt", "Frankfurt am Main"),
|
|
("Göttingen", "Goettingen", "Gottingen"),
|
|
("Würzburg", "Wuerzburg", "Wurzburg"),
|
|
("Tübingen", "Tuebingen", "Tubingen"),
|
|
("Lübeck", "Luebeck", "Lubeck"),
|
|
("Münster", "Muenster", "Munster"),
|
|
("Saarbrücken", "Saarbruecken", "Saarbrucken"),
|
|
("Neukirchen-Vluyn",),
|
|
# Austria
|
|
("Vienna", "Wien", "Vienne"),
|
|
("Salzburg",),
|
|
("Innsbruck",),
|
|
# Switzerland
|
|
("Zurich", "Zürich", "Zuerich"),
|
|
("Geneva", "Genève", "Geneve", "Genf", "Ginevra"),
|
|
("Basel", "Basle", "Bâle"),
|
|
("Bern", "Berne"),
|
|
("Lucerne", "Luzern"),
|
|
("St. Gallen", "Sankt Gallen", "Saint Gall"),
|
|
# The rest of Europe
|
|
("Prague", "Praha", "Prag"),
|
|
("Warsaw", "Warszawa", "Warschau"),
|
|
("Krakow", "Kraków", "Cracow"),
|
|
("Wroclaw", "Wrocław", "Breslau"),
|
|
("Gdansk", "Gdańsk", "Danzig"),
|
|
("Poznan", "Poznań", "Posen"),
|
|
("Lisbon", "Lisboa", "Lissabon"),
|
|
("Milan", "Milano", "Mailand"),
|
|
("Rome", "Roma", "Rom"),
|
|
("Turin", "Torino"),
|
|
("Florence", "Firenze"),
|
|
("Naples", "Napoli", "Neapel"),
|
|
("Venice", "Venezia", "Venedig"),
|
|
("Genoa", "Genova"),
|
|
("Brussels", "Bruxelles", "Brussel", "Brüssel"),
|
|
("Antwerp", "Antwerpen", "Anvers"),
|
|
("Ghent", "Gent", "Gand"),
|
|
("Bruges", "Brugge"),
|
|
("Liège", "Liege", "Luik", "Lüttich"),
|
|
("The Hague", "Den Haag", "s-Gravenhage"),
|
|
("Luxembourg", "Luxemburg"),
|
|
("Seville", "Sevilla"),
|
|
("Athens", "Athina", "Athen"),
|
|
("Bucharest", "București", "Bucuresti"),
|
|
("Belgrade", "Beograd"),
|
|
("Bratislava", "Pressburg"),
|
|
("Kyiv", "Kiev"),
|
|
("Cluj-Napoca", "Cluj"),
|
|
("Timișoara", "Timisoara"),
|
|
("Marseille", "Marseilles"),
|
|
("Bolzano", "Bozen"),
|
|
("Padua", "Padova"),
|
|
("Forlì",),
|
|
("L'Aquila",),
|
|
("Massa-Carrara", "Massa Carrara"),
|
|
# France, with the marks and hyphens a fold drops
|
|
("Aix-en-Provence",), ("Bourg-en-Bresse",), ("Boulogne-sur-Mer",), ("Chambéry",),
|
|
("Clermont-Ferrand",), ("La Roche-sur-Yon",), ("Les Sables-d'Olonne",), ("Loudéac",),
|
|
("Nîmes",), ("Orléans",), ("Rezé",), ("Saint-Brieuc",), ("Saint-Étienne",),
|
|
("Saint-Herblain",), ("Saint-Malo",), ("Saint-Nazaire",), ("Saint-Ouen-l'Aumône",),
|
|
# Germany, likewise
|
|
("Annaberg-Buchholz",), ("Baden-Baden",), ("Bietigheim-Bissingen",), ("Böblingen",),
|
|
("Donauwörth",), ("Flörsheim", "Flohrsheim"), ("Fürth",), ("Gießen", "Giessen"),
|
|
("Göppingen",), ("Großostheim",), ("Günzburg",), ("Güstrow",), ("Halle (Saale)",),
|
|
("Höxter",), ("Idar-Oberstein",), ("Königsbrunn",), ("Künzelsau",), ("Lörrach",),
|
|
("Lüchow",), ("Meißen", "Meissen"), ("Möhnesee",), ("Mönchengladbach",), ("Mühldorf",),
|
|
("Mülheim",), ("Neumünster",), ("Nördlingen",), ("Öhringen",), ("Ribnitz-Damgarten",),
|
|
("Röbel",), ("Schwäbisch Gmünd",), ("Schwäbisch Hall",), ("Überlingen",),
|
|
("Villingen-Schwenningen",), ("Waren an der Müritz",), ("Wolfenbüttel",),
|
|
# Elsewhere
|
|
("Cegléd",), ("Brașov", "Brasov"), ("Älmhult",), ("Hornbæk",), ("Melaka", "Malacca"),
|
|
("Samut Prakan", "Samut Prakarn"), ("Xi'an",), ("Ho Chi Minh City", "Ho Chi Minh"),
|
|
)
|
|
|
|
|
|
def _variants(text: str) -> tuple[str, ...]:
|
|
"""A fold and its `aa` spelling: Danish writes `å` as `aa` (`Aarhus`, `Taastrup`)
|
|
and `Århus` and `Tåstrup` fold to `arhus` and `tastrup`."""
|
|
return tuple(dict.fromkeys((text, text.replace("aa", "a"))))
|
|
|
|
|
|
def _aliases() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]:
|
|
"""Every folded spelling to its whole group (all folded), and to its shown
|
|
spelling. The group starts with the first spelling listed."""
|
|
groups: dict[str, tuple[str, ...]] = {}
|
|
shown: dict[str, str] = {}
|
|
for group in _ALIAS_GROUPS:
|
|
members = tuple(dict.fromkeys(v for name in group for v in _variants(fold(name))))
|
|
for name in group:
|
|
shown.setdefault(fold(name), name)
|
|
for member in members:
|
|
groups[member] = members
|
|
return groups, shown
|
|
|
|
|
|
_GROUPS, _SHOWN = _aliases()
|
|
|
|
|
|
def _expand(text: str) -> tuple[str, ...]:
|
|
"""A folded town and everything it is also written as."""
|
|
return _GROUPS.get(text) or _variants(text)
|
|
|
|
|
|
# -------------------------------------------------------------------- regions
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Region:
|
|
"""A region below a country, or across two. `names` are what a board calls
|
|
the region itself, `towns` the places in it; a line is in the region when it
|
|
has one of either as whole words, or is in a region it `includes`, by key.
|
|
Only `names` are what the region is suggested and read as. Spellings are
|
|
shown with their marks and folded when used."""
|
|
|
|
key: str
|
|
label: str
|
|
names: tuple[str, ...]
|
|
towns: tuple[str, ...]
|
|
country: str = ""
|
|
includes: tuple[str, ...] = ()
|
|
|
|
|
|
def _split(text: str) -> tuple[str, ...]:
|
|
return tuple(part.strip() for part in text.split(",") if part.strip())
|
|
|
|
|
|
_SKANE = Region(
|
|
"skane", "Skåne",
|
|
_split("Skåne, Skåne län, Skåne County, Region Skåne, Skåne Region, Scania, Skaane"),
|
|
_split("""Malmö, Lund, Helsingborg, Kristianstad, Landskrona, Trelleborg, Ängelholm,
|
|
Hässleholm, Ystad, Eslöv, Höganäs, Staffanstorp, Lomma, Bjuv, Burlöv, Kävlinge,
|
|
Svedala, Vellinge, Simrishamn, Tomelilla, Sjöbo, Svalöv, Klippan, Åstorp,
|
|
Bromölla, Osby, Perstorp, Örkelljunga, Östra Göinge, Hörby, Höör, Båstad,
|
|
Skurup, Arlöv, Limhamn, Dalby, Åkarp, Bjärred, Höllviken, Ljungbyhed,
|
|
Löddeköpinge, Furulund, Genarp, Veberöd, Skanör, Falsterbo, Mölle, Arild"""),
|
|
"SE")
|
|
|
|
_HOVEDSTADEN = Region(
|
|
"hovedstaden", "Capital Region of Denmark",
|
|
_split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"),
|
|
_split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup,
|
|
Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte,
|
|
Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre,
|
|
Høje-Taastrup, Hørsholm, Ishøj, Lyngby-Taarbæk, Rudersdal, Rødovre, Tårnby,
|
|
Vallensbæk, Kgs. Lyngby, Kongens Lyngby, Lyngby, Virum, Holte, Nærum, Birkerød,
|
|
Vedbæk, Skodsborg, Klampenborg, Charlottenlund, Hellerup, Ordrup, Bagsværd,
|
|
Søborg, Mørkhøj, Måløv, Skovlunde, Smørum, Ølstykke, Stenløse, Værløse, Farum,
|
|
Slangerup, Hedehusene, Taastrup, Kastrup, Amager, Valby, Vanløse, Brønshøj,
|
|
Nørrebro, Østerbro, Vesterbro, Nordhavn, Ørestad, Sydhavn, Islands Brygge,
|
|
Refshaleøen, Humlebæk, Kokkedal, Espergærde, Snekkersten, Rungsted, Rungsted Kyst,
|
|
Lillerød, Frederiksværk, Hundested, Helsinge, Gilleleje, Græsted, Vallensbæk Strand,
|
|
Skævinge, Rønne, Nexø, Allinge, Svaneke, Aakirkeby, Hasle"""),
|
|
"DK")
|
|
|
|
# Region Sjælland, not the island: Copenhagen is on Zealand too, but a board that
|
|
# writes "Region Zealand" means the region and the Capital Region has its own key.
|
|
# The bare word is left out of `names` since "New Zealand" would match it.
|
|
_ZEALAND = Region(
|
|
"zealand", "Zealand",
|
|
_split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"),
|
|
_split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved,
|
|
Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg,
|
|
Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing,
|
|
Haslev, Korsør, Skælskør, Præstø, Stege, Fakse, Rødby, Rødbyhavn, Jyllinge,
|
|
Viby Sjælland, Solrød Strand, Fuglebjerg, Gørlev, Dianalund, Jyderup, Hvalsø,
|
|
Tølløse, Hørve, Fårevejle, Bjæverskov, Nørre Asmindrup"""),
|
|
"DK")
|
|
|
|
_JUTLAND = Region(
|
|
"jutland", "Jutland",
|
|
_split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland,
|
|
North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland,
|
|
East Jutland, West Jutland, Vestjylland, Østjylland"""),
|
|
_split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning,
|
|
Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov,
|
|
Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå,
|
|
Hjørring, Frederikshavn, Skagen, Thisted, Struer, Ikast, Brande, Varde, Ribe,
|
|
Tønder, Vojens, Hadsten, Hobro, Skanderborg, Odder, Hedensted, Vejen, Ebeltoft,
|
|
Lemvig, Rønde, Egå, Lystrup, Hinnerup, Tilst, Hammel, Galten, Sabro, Harlev,
|
|
Løsning, Højbjerg, Juelsminde, Nørresundby, Brønderslev, Aars, Nibe, Sæby, Løgstør, Tarm,
|
|
Skjern, Padborg, Rødekro, Toftlund, Bramming, Kruså, Brabrand, Ry, Gjern"""),
|
|
"DK")
|
|
|
|
_FUNEN = Region(
|
|
"funen", "Funen",
|
|
_split("Funen, Fyn, Region Fyn"),
|
|
_split("""Odense, Svendborg, Nyborg, Faaborg, Fåborg, Middelfart, Assens, Kerteminde,
|
|
Bogense, Langeskov, Ringe, Otterup, Munkebo, Vissenbjerg, Årslev, Tommerup,
|
|
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
|
|
"DK")
|
|
|
|
# The Øresund region is the two capital regions either side of the sound. It is
|
|
# suggested on its own names only; a line in either region is in it.
|
|
_ORESUND = Region(
|
|
"oresund", "Øresund",
|
|
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"),
|
|
(), includes=("hovedstaden", "skane"))
|
|
|
|
# Germany's states and the areas boards write for a remote role's area, by
|
|
# name, towns not listed: a town in Germany is found through the country.
|
|
_GERMANY = tuple(Region(key, label, _split(names), (), "DE") for key, label, names in (
|
|
("niedersachsen", "Niedersachsen", "Niedersachsen, Lower Saxony"),
|
|
("norddeutschland", "Norddeutschland", "Norddeutschland, Northern Germany"),
|
|
("ostdeutschland", "Ostdeutschland", "Ostdeutschland, Eastern Germany"),
|
|
("suddeutschland", "Süddeutschland", "Süddeutschland, Southern Germany"),
|
|
("baden-wurttemberg", "Baden-Württemberg", "Baden-Württemberg"),
|
|
("bayern", "Bayern", "Bayern, Bavaria"),
|
|
("brandenburg", "Brandenburg", "Brandenburg"),
|
|
("hessen", "Hessen", "Hessen, Hesse"),
|
|
("mecklenburg-vorpommern", "Mecklenburg-Vorpommern",
|
|
"Mecklenburg-Vorpommern, Mecklenburg-Western Pomerania"),
|
|
("nordrhein-westfalen", "Nordrhein-Westfalen", "Nordrhein-Westfalen, North Rhine-Westphalia, NRW"),
|
|
("rheinland-pfalz", "Rheinland-Pfalz", "Rheinland-Pfalz, Rhineland-Palatinate"),
|
|
("saarland", "Saarland", "Saarland"),
|
|
("sachsen", "Sachsen", "Sachsen, Saxony"),
|
|
("sachsen-anhalt", "Sachsen-Anhalt", "Sachsen-Anhalt, Saxony-Anhalt"),
|
|
("schleswig-holstein", "Schleswig-Holstein", "Schleswig-Holstein"),
|
|
("thuringen", "Thüringen", "Thüringen, Thuringia"),
|
|
("bodensee", "Bodensee", "Bodensee, Lake Constance"),
|
|
("harz", "Harz", "Harz"),
|
|
("rugen", "Rügen", "Rügen"),
|
|
("uckermark", "Uckermark", "Uckermark")))
|
|
|
|
REGIONS: dict[str, Region] = {
|
|
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN, *_GERMANY)}
|
|
|
|
|
|
def _phrases(region: Region) -> tuple[str, ...]:
|
|
"""Every folded phrase that puts a line in the region, with the other
|
|
spellings of its towns, and those of the regions it includes."""
|
|
found: dict[str, None] = {}
|
|
for text in (*region.names, *region.towns):
|
|
for form in _expand(fold(text)):
|
|
found[form] = None
|
|
for key in region.includes:
|
|
found.update(dict.fromkeys(_phrases(REGIONS[key])))
|
|
return tuple(found)
|
|
|
|
|
|
_PHRASES: dict[str, tuple[str, ...]] = {key: _phrases(r) for key, r in REGIONS.items()}
|
|
|
|
#: A folded key, label or name to the region key; a name in two regions ("Capital
|
|
#: Region of Denmark" is also in Øresund) belongs to the one that lists it first.
|
|
_REGION_BY_NAME: dict[str, str] = {}
|
|
for _key, _region in REGIONS.items():
|
|
for _name in (_region.key, _region.label, *_region.names):
|
|
_REGION_BY_NAME.setdefault(fold(_name), _key)
|
|
|
|
#: What a place is shown as when a chip is read from the address: every alias
|
|
#: spelling, every region and every town of one, as written above.
|
|
_TOWN_LABELS: dict[str, str] = dict(_SHOWN)
|
|
for _region in REGIONS.values():
|
|
for _town in (_region.label, *_region.towns, *_region.names):
|
|
_TOWN_LABELS.setdefault(fold(_town), _town)
|
|
|
|
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
|
|
|
|
|
|
def _group(folded: str) -> str:
|
|
"""The key of the town a fold is: the first spelling of its alias group, else
|
|
the fold with `aa` as `a`."""
|
|
return _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
|
|
|
|
|
|
def _known_towns() -> dict[str, str]:
|
|
"""Every town the tables name, as a fold, to its group: the alias groups, the
|
|
towns of the regions and the cities `country_of` knows."""
|
|
names = [*_GROUPS, *(fold(city) for city in _CITIES),
|
|
*(fold(town) for region in REGIONS.values() for town in region.towns)]
|
|
return {form: _group(form) for name in names for form in _variants(name)}
|
|
|
|
|
|
_KNOWN = _known_towns()
|
|
_KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN)
|
|
|
|
|
|
# --------------------------------------------------------------------- tokens
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Token:
|
|
"""One place filter. `value` is a country code, a region key or a fold;
|
|
for `remote`, the place token (`country:DK`), `anywhere` or `any`."""
|
|
|
|
kind: str
|
|
value: str
|
|
label: str
|
|
|
|
def __str__(self) -> str:
|
|
return f"{self.kind}:{self.value}"
|
|
|
|
|
|
def _region_key(text: str) -> str:
|
|
folded = fold(text)
|
|
return _REGION_BY_NAME.get(folded, folded)
|
|
|
|
|
|
#: The words that are Anywhere: in a remote token (`remote:anywhere`), and in
|
|
#: a place text that says remote ("Home Based - Worldwide").
|
|
ANYWHERE_WORDS = frozenset(("anywhere", "worldwide", "world wide"))
|
|
|
|
|
|
def _remote(value: str) -> Token:
|
|
"""`remote:<place token>`: `value` is the place token, whose label the
|
|
chip reads as "Remote · Denmark"; Europe (`country:EU`) is "Remote ·
|
|
Europe", `anywhere` "Remote · anywhere" and `any` "Remote". A remote
|
|
token inside one names nothing."""
|
|
if fold(value) == "any":
|
|
return Token("remote", "any", "Remote")
|
|
if fold(value) in ANYWHERE_WORDS:
|
|
return Token("remote", "anywhere", "Remote · anywhere")
|
|
inner = parse(value)
|
|
if inner.kind == "remote" or not value:
|
|
return Token("remote", "", "Remote")
|
|
shown = "Europe" if (inner.kind, inner.value) == ("country", "EU") else inner.label
|
|
return Token("remote", str(inner), f"Remote · {shown}")
|
|
|
|
|
|
def parse(token: str) -> Token:
|
|
"""`country:DK`, `region:skane`, `place:malmo` or `remote:<one of these>`.
|
|
Bare text, or a prefix that is none of these, is a place: `Skåne` is
|
|
`place:skane`. A country by name (`country:Denmark`) and a region by label
|
|
(`region:Skåne`) are read as well."""
|
|
kind, sep, value = (token or "").partition(":")
|
|
kind = kind.strip().lower()
|
|
if sep and kind == "remote":
|
|
return _remote(value.strip())
|
|
if not sep or kind not in ("country", "region", "place"):
|
|
kind, value = "place", token or ""
|
|
value = value.strip()
|
|
if kind == "country":
|
|
code = value.upper()
|
|
if code not in _COUNTRY_LABELS:
|
|
code = _COUNTRY_BY_NAME.get(fold(value), code)
|
|
return Token("country", code, _COUNTRY_LABELS.get(code, (code,))[0])
|
|
if kind == "region":
|
|
key = _region_key(value)
|
|
return Token("region", key, REGIONS[key].label if key in REGIONS else key)
|
|
folded = fold(value)
|
|
return Token("place", folded, _TOWN_LABELS.get(folded) or folded.title())
|
|
|
|
|
|
def label(token: str | Token) -> str:
|
|
"""The text of a chip read from the address: `country:DK` is `Denmark`,
|
|
`region:skane` is `Skåne`, `place:malmo` is `Malmö`, `remote:country:DK`
|
|
is `Remote · Denmark`. A town the tables do not know is its fold in title
|
|
case."""
|
|
return (token if isinstance(token, Token) else parse(token)).label
|
|
|
|
|
|
# ------------------------------------------------------------------- matching
|
|
|
|
SQL_FUNCTION = "place_match"
|
|
|
|
|
|
def known(token: str) -> bool:
|
|
"""Whether the token names something: a country of the table (or `EU`), a
|
|
region of `REGIONS`, a place with a value, or `remote:` and one of these,
|
|
`anywhere` or `any`. `region:atlantis`, `country:ZZ`, `place:` and
|
|
`remote:country:ZZ` are not."""
|
|
return _known(parse(token))
|
|
|
|
|
|
def _known(tok: Token) -> bool:
|
|
if tok.kind == "remote":
|
|
return tok.value in ("any", "anywhere") or (bool(tok.value) and _known(parse(tok.value)))
|
|
if tok.kind == "country":
|
|
return tok.value in _COUNTRY_LABELS or tok.value == "EU"
|
|
if tok.kind == "region":
|
|
return tok.value in REGIONS
|
|
return bool(tok.value)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _Matcher:
|
|
"""What a token asks of a role. `phrases` are whole-word phrases of the
|
|
folded place; `countries` are `(code, mentions)`: the role's own country, or
|
|
a line with one of the mentions while `country` is empty or `EU`, since a
|
|
board listing several offices has no single country."""
|
|
|
|
phrases: frozenset[str]
|
|
countries: tuple[tuple[str, frozenset[str]], ...]
|
|
longest: int
|
|
|
|
|
|
@lru_cache(maxsize=1024)
|
|
def _matcher(token: str) -> _Matcher | None:
|
|
"""The token read once: `None` when it names nothing.
|
|
|
|
A `place:` value that is also a region or a country is matched as that too:
|
|
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
|
|
written as a bare word means what the owner meant by it."""
|
|
tok = parse(token)
|
|
if not _known(tok) or tok.kind == "remote":
|
|
return None
|
|
phrases: set[str] = set()
|
|
countries: list[tuple[str, frozenset[str]]] = []
|
|
if tok.kind == "country":
|
|
countries.append((tok.value, frozenset(_COUNTRY_MENTIONS.get(tok.value, ()))))
|
|
elif tok.kind == "region":
|
|
phrases.update(_PHRASES[tok.value])
|
|
else:
|
|
phrases.update(_expand(tok.value))
|
|
phrases.update(_PHRASES.get(_REGION_BY_NAME.get(tok.value, ""), ()))
|
|
if code := _COUNTRY_BY_NAME.get(tok.value):
|
|
countries.append((code, frozenset(_COUNTRY_MENTIONS.get(code, ()))))
|
|
phrases.discard("")
|
|
longest = max((len(p.split()) for p in (*phrases, *(m for _, ms in countries for m in ms))),
|
|
default=1)
|
|
return _Matcher(frozenset(phrases), tuple(countries), longest)
|
|
|
|
|
|
@lru_cache(maxsize=32768)
|
|
def _runs(place_fold: str, longest: int) -> frozenset[str]:
|
|
"""Every run of up to `longest` words of a folded line, each a whole-word
|
|
phrase the line holds."""
|
|
words = place_fold.split()
|
|
return frozenset(" ".join(words[i:i + n])
|
|
for n in range(1, longest + 1) for i in range(len(words) - n + 1))
|
|
|
|
|
|
def matches(place_fold: str, country: str, token: str) -> bool:
|
|
"""Whether a role whose folded place and country are these is found by the
|
|
token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or
|
|
a Danish city) while `country` is empty or `EU`. `region:skane` is any of
|
|
its names or towns as whole words, `place:malmo` the town and its aliases.
|
|
A token that names nothing (`known` is false) matches no role, nor does a
|
|
`remote:` token, which only the gazetteer's areas answer."""
|
|
match = _matcher(token)
|
|
if match is None:
|
|
return False
|
|
runs = None
|
|
if match.phrases:
|
|
runs = _runs(place_fold, match.longest)
|
|
if not match.phrases.isdisjoint(runs):
|
|
return True
|
|
for code, mentions in match.countries:
|
|
if country == code:
|
|
return True
|
|
if mentions and country in ("", "EU"):
|
|
runs = _runs(place_fold, match.longest) if runs is None else runs
|
|
if not mentions.isdisjoint(runs):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _place_match(place_fold: str | None, country: str | None, token: str) -> int:
|
|
return int(matches(place_fold or "", country or "", token))
|
|
|
|
|
|
def register(conn: sqlite3.Connection) -> None:
|
|
"""Give a connection `place_match(place_fold, country, token)`, which is 1 when
|
|
`matches` is true. `match_sql` returns a clause that calls it."""
|
|
conn.create_function(SQL_FUNCTION, 3, _place_match, deterministic=True)
|
|
|
|
|
|
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
|
|
-> tuple[str, list]:
|
|
"""SQL over the two columns for one token, with one `?` parameter. It calls
|
|
`place_match`, so the connection needs `register`. Several tokens are joined
|
|
with OR by the caller. Bare text is a place. A token that names nothing
|
|
(`known` is false) is `0`, which matches no role."""
|
|
tok = parse(token)
|
|
if not _known(tok) or tok.kind == "remote":
|
|
return "0", []
|
|
return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)]
|
|
|
|
|
|
# ---------------------------------------------------------------- suggestions
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Suggestion:
|
|
"""`token` is what the chip holds, `kind` is `country`, `region`, `place`
|
|
or `remote`, `count` the open roles it stands for."""
|
|
|
|
token: str
|
|
label: str
|
|
kind: str
|
|
count: int
|
|
|
|
|
|
def _prefixes(q: str, candidates: Iterable[str]) -> bool:
|
|
"""`q` starts a word of a candidate, or a run of words: `kgs ly` finds
|
|
`kgs lyngby`, `hague` finds `the hague`."""
|
|
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
|
|
|
|
|
|
#: Where one town ends in a line: `;` and `|` between places, and `or` / `eller` /
|
|
#: `og` between two towns in one ("København V eller Aalborg").
|
|
_PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
|
|
|
|
#: Danish boards append "and the possibility of working from home".
|
|
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
|
|
|
|
#: A slash between two places, "Malmö/Lund", "Berlin / Munich".
|
|
_SLASH = re.compile(r"\s*/\s*")
|
|
|
|
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
|
|
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
|
|
|
|
#: A country or "Remote" ahead of the town, "Sweden - Lund", "Remote-New Zealand".
|
|
_LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$")
|
|
|
|
|
|
def remote_part(text: str) -> bool:
|
|
"""Whether a place text, or one part of it, says the role is worked
|
|
remotely: "Germany (Remote)", "Remote - US", "Home Based - EMEA",
|
|
"Worldwide". One that says hybrid too is not ("Remote or hybrid,
|
|
Berlin"), as `remote_of` reads hybrid first."""
|
|
return remote_of(text) == "remote"
|
|
|
|
|
|
def _towns(place: str) -> list[str]:
|
|
"""The first comma part of each place of a line: `Aarhus N, Region Central
|
|
Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal
|
|
district is dropped, since `place:kobenhavn` finds `København K` anyway.
|
|
A part that says remote (`remote_part`) names where a hire may live, not
|
|
a town: `USA (Remote)` and `Remote Germany` give none."""
|
|
found = []
|
|
for part in _PART.split(_AND_HOME.sub("", place or "")):
|
|
if remote_part(part):
|
|
continue
|
|
head = part.split(",")[0].strip()
|
|
if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN:
|
|
head = lead[2]
|
|
head = _DISTRICT.sub("", head)
|
|
if fold(head) and fold(head) not in _NOT_A_TOWN:
|
|
found.append(head)
|
|
return found
|
|
|
|
|
|
def _towns_in(folded: str) -> list[str]:
|
|
"""The known towns a fold holds as whole words, longest first from the left,
|
|
as their groups: `munich berlin` is Munich and Berlin, `copenhagen office` is
|
|
Copenhagen, `hoje taastrup` is one town and not also Taastrup."""
|
|
words = folded.split()
|
|
found: list[str] = []
|
|
i = 0
|
|
while i < len(words):
|
|
for n in range(min(_KNOWN_LONGEST, len(words) - i), 0, -1):
|
|
if (group := _KNOWN.get(" ".join(words[i:i + n]))) is not None:
|
|
found.append(group)
|
|
i += n
|
|
break
|
|
else:
|
|
i += 1
|
|
return list(dict.fromkeys(found))
|
|
|
|
|
|
def towns(place: str) -> frozenset[str]:
|
|
"""The towns a line names, each as its group's key, so `København K` and
|
|
`Copenhagen, Denmark` are both `{"copenhagen"}`: the first comma part of each
|
|
place, and every known town the line holds as whole words, so a list such as
|
|
`Aalborg, Copenhagen, Kolding` gives all three. A state or province after
|
|
the town is not a town, so `Mountain View, California` and `San Francisco,
|
|
California` share none. Empty for a line that names only a country, a region
|
|
or how the role is worked."""
|
|
found = {_group(fold(town)) for town in _towns(place)}
|
|
found.update(_towns_in(fold(place or "")))
|
|
return frozenset(found)
|
|
|
|
|
|
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
|
|
"""Open roles per region, over folded lines. A line is cut into every run of
|
|
words once, and each region asks whether it holds one of its phrases."""
|
|
sets = {key: frozenset(_PHRASES[key]) for key in keys}
|
|
longest = max((len(p.split()) for phrases in sets.values() for p in phrases), default=1)
|
|
counts: Counter[str] = Counter()
|
|
for text, roles in places:
|
|
runs = _runs(text, longest)
|
|
for key, phrases in sets.items():
|
|
if not phrases.isdisjoint(runs):
|
|
counts[key] += roles
|
|
return counts
|
|
|
|
|
|
#: The kinds a remote suggestion is offered for by name alone; a region or a
|
|
#: town only when a remote area names it or a place in it.
|
|
_WIDE = ("continent", "country")
|
|
|
|
|
|
def _asks_remote(text: str) -> tuple[bool, str]:
|
|
"""Whether typed text asks for the remote tokens as such, and the text of
|
|
a place after "remote": `remo` is `(True, "")`, `remote den` `(True,
|
|
"den")`, `den` `(False, "den")`. Three letters at least, so "re" is
|
|
still Regensburg."""
|
|
head, _, rest = text.partition(" ")
|
|
if head == "remote":
|
|
return True, rest.strip()
|
|
return (True, "") if len(text) >= 3 and "remote".startswith(text) else (False, text)
|
|
|
|
|
|
def _remote_suggestions(text: str, gaz: Gazetteer,
|
|
areas: list[tuple[tuple[int, ...], int]]) -> list[Suggestion]:
|
|
"""`remote:` tokens for typed text, each with the open remote roles it
|
|
finds (`Gazetteer.overlap`, which holds Anywhere), none that finds
|
|
nothing, widest first and then by open roles. A place is offered when it
|
|
is a continent or a country, or a remote area names it or a place in it
|
|
(`held`). "remote" alone offers every continent and country held;
|
|
"remote den" or "den" the places offered whose name it starts, a region
|
|
or a town only when its token names it alone. `remote:anywhere` is a
|
|
token and no suggestion: every remote chip holds Anywhere already."""
|
|
asked, text = _asks_remote(text)
|
|
held: set[int] = set()
|
|
for ids, _ in areas:
|
|
for place_id in ids:
|
|
held |= {place_id, *gaz.ancestors(place_id)}
|
|
if asked and not text:
|
|
candidates = {p for p in held if gaz.places[p].kind in _WIDE}
|
|
elif text:
|
|
candidates = {p for p, place in gaz.places.items()
|
|
if (place.kind in _WIDE or p in held) and _prefixes(text, gaz.names(p))}
|
|
else:
|
|
candidates = set()
|
|
found: dict[str, tuple[int, Suggestion]] = {}
|
|
for place_id in candidates - {gaz.anywhere}:
|
|
token = parse(f"remote:{gaz.token(place_id)}")
|
|
named = gaz.lookup(str(token))
|
|
if str(token) in found or not named or place_id not in named \
|
|
or (token.value.startswith("place:") and len(named) > 1):
|
|
continue
|
|
wanted = gaz.overlap(named)
|
|
if count := sum(n for ids, n in areas if not wanted.isdisjoint(ids)):
|
|
found[str(token)] = (KINDS.index(gaz.places[place_id].kind),
|
|
Suggestion(str(token), token.label, "remote", count))
|
|
return [s for _, s in sorted(found.values(), key=lambda f: (f[0], -f[1].count, f[1].label))]
|
|
|
|
|
|
def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12,
|
|
remote: Iterable[tuple[Iterable[int], int]] = (),
|
|
gaz: Gazetteer | None = None,
|
|
remote_any: Callable[[], int] | None = None) -> list[Suggestion]:
|
|
"""Tokens for the text typed so far. `places` is `(place, country, open
|
|
roles)`, `remote` is `(area, open roles)` for the open remote roles, the
|
|
area as place ids of `gaz`, and `remote_any` counts the open roles
|
|
`remote:any` finds for the agent asking, called only when it is offered.
|
|
`remote:any` ("Remote") first when `q` starts "remote", then countries,
|
|
then the other remote tokens, then regions, then towns, each by open
|
|
roles.
|
|
|
|
A country is suggested when `q` starts a word of one of its names, with the
|
|
roles that carry its code. A region likewise on its label and names, with the
|
|
roles whose line is in it. A town is the first comma part of each `;` part of
|
|
a line, grouped by fold and by alias (`Copenhagen` and `København` are one
|
|
town, shown as its commonest spelling). Countries, regions and towns with no
|
|
open role are left out: a chip that finds nothing helps nobody. Remote
|
|
tokens are `_remote_suggestions`, offered only with `gaz`."""
|
|
text = fold(q)
|
|
if not text:
|
|
return []
|
|
rows = [(place or "", country or "", int(roles)) for place, country, roles in places]
|
|
|
|
by_country: Counter[str] = Counter()
|
|
for _, country, roles in rows:
|
|
by_country[country] += roles
|
|
countries = [Suggestion(f"country:{code}", names[0], "country", by_country[code])
|
|
for code, names in _COUNTRY_LABELS.items()
|
|
if by_country[code] and _prefixes(text, _COUNTRY_FOLDS[code])]
|
|
|
|
named = [key for key, r in REGIONS.items()
|
|
if _prefixes(text, [fold(r.label), *map(fold, r.names)])]
|
|
counts = _region_counts(named, [(fold(place), roles) for place, _, roles in rows])
|
|
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
|
|
for key in named if counts[key]]
|
|
|
|
# A line's town is a group. A line that is a town is that group's spelling;
|
|
# one that only holds a known town ("Copenhagen Office", "Munich - Berlin")
|
|
# counts under each such town and shows as none; anything else is its own.
|
|
counts_by_town: Counter[str] = Counter()
|
|
spellings: dict[str, Counter[str]] = {}
|
|
written: dict[str, set[str]] = {}
|
|
for place, _, roles in rows:
|
|
for town in _towns(place):
|
|
folded = fold(town)
|
|
if folded not in _KNOWN and (inside := _towns_in(folded)):
|
|
for group in inside:
|
|
counts_by_town[group] += roles
|
|
continue
|
|
group = _group(folded)
|
|
counts_by_town[group] += roles
|
|
spellings.setdefault(group, Counter())[town] += roles
|
|
written.setdefault(group, set()).add(folded)
|
|
towns = []
|
|
for group, total in counts_by_town.items():
|
|
forms = {group, *written.get(group, ())}
|
|
if not _prefixes(text, [form for folded in forms for form in _expand(folded)]):
|
|
continue
|
|
seen = spellings.get(group)
|
|
shown = min(seen, key=lambda s: (-seen[s], s)) if seen \
|
|
else _TOWN_LABELS.get(group) or group.title()
|
|
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total))
|
|
|
|
remotes = _remote_suggestions(text, gaz, [(tuple(ids), int(n)) for ids, n in remote]) \
|
|
if gaz is not None else []
|
|
first = []
|
|
if remote_any is not None and _asks_remote(text)[0] and (count := remote_any()):
|
|
first.append(Suggestion("remote:any", "Remote", "remote", count))
|
|
|
|
countries.sort(key=lambda s: (-s.count, s.label))
|
|
regions.sort(key=lambda s: (-s.count, s.label))
|
|
towns.sort(key=lambda s: (-s.count, s.label))
|
|
return (first + countries + remotes + regions + towns)[:limit]
|
|
|
|
|
|
# ------------------------------------------------------------------ gazetteer
|
|
|
|
#: The kinds of place, widest first: the index is how specific a kind is.
|
|
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
|
|
|
|
#: `(kind, key)` of Anywhere, where a remote hire may live when a posting says
|
|
#: "worldwide". A continent with no parent and nothing in it, so `under` never
|
|
#: reaches every role.
|
|
ANYWHERE = ("continent", "world")
|
|
|
|
#: What is Anywhere in a part of a place text that says remote: "Remote,
|
|
#: Global" is Anywhere, while "Global HQ, Berlin" is Berlin.
|
|
_ANYWHERE_IF_REMOTE = frozenset(("global", "international", *ANYWHERE_WORDS))
|
|
|
|
#: Entries of `_CITIES` that name a region rather than a town.
|
|
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
|
|
"bay area", "emilia romagna", "tuscany", "lazio"))
|
|
|
|
#: Words a place text holds when it names how or where-ever a role is worked
|
|
#: rather than a place: "Remote", "Multiple Locations", "Home based". A text
|
|
#: of these alone is no place, so it is never listed as unresolved.
|
|
_PLACELESS = frozenset((
|
|
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
|
|
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
|
|
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
|
|
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
|
|
|
|
|
|
def placeless(text: str) -> bool:
|
|
"""Whether a place text names no place at all: empty, or only numbers and
|
|
words such as `Remote`, `Hybrid`, `2 Locations`."""
|
|
return all(word in _PLACELESS or word.isdigit() for word in fold(text).split())
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Seed:
|
|
"""One place as the code knows it: `key` is unique within `kind` (an ISO
|
|
code for a country, a region's key, a town's group), `parent` the
|
|
`(kind, key)` of the place it lies in, `aliases` the folds it is written
|
|
as."""
|
|
|
|
kind: str
|
|
key: str
|
|
name: str
|
|
parent: tuple[str, str] | None
|
|
aliases: tuple[str, ...]
|
|
|
|
|
|
def _subdivisions() -> list[tuple[str, str, str]]:
|
|
"""`(country, code, name)` of every US state and Canadian province."""
|
|
found = []
|
|
for line in _SUBDIVISIONS.split("\n"):
|
|
if line:
|
|
country, rest = line.split(" ", 1)
|
|
for entry in rest.split("|"):
|
|
name, code = entry.strip().rsplit(" ", 1)
|
|
found.append((country, code, name))
|
|
return found
|
|
|
|
|
|
def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
|
|
"""The gazetteer the dictionaries in code describe, parents before their
|
|
children, and the inclusions beyond the tree (Øresund holds the Capital
|
|
Region and Skåne, which lie in two countries).
|
|
|
|
Europe is a continent over the countries of `_EUROPE`, and is also what
|
|
`EU`, `EMEA` and `European Union` name. Asia Pacific (`APAC`) is a continent over
|
|
nothing. Every country of `_COUNTRY_NAMES`
|
|
by its names; the US states and Canadian provinces as regions; the
|
|
regions of `REGIONS`; the towns of the alias groups, of the regions and
|
|
of `_CITIES`, one place per group (`Copenhagen`, `København`,
|
|
`Kopenhamn` are one), in their region when a region lists them and else
|
|
in their country. Anywhere (`ANYWHERE`) stands apart, with no alias."""
|
|
europe = ("continent", "europe")
|
|
seeds: dict[tuple[str, str], Seed] = {}
|
|
|
|
def add(kind: str, key: str, name: str, parent: tuple[str, str] | None,
|
|
aliases: Iterable[str]) -> None:
|
|
folds = tuple(dict.fromkeys(a for a in aliases if a))
|
|
if (kind, key) in seeds:
|
|
old = seeds[(kind, key)]
|
|
folds = tuple(dict.fromkeys((*old.aliases, *folds)))
|
|
seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds)
|
|
else:
|
|
seeds[(kind, key)] = Seed(kind, key, name, parent, folds)
|
|
|
|
add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS)))
|
|
# Asia Pacific holds no country: a role placed there alone keeps the
|
|
# host's country, and one beside a country is that country.
|
|
add("continent", "asia-pacific", "Asia Pacific", None, ("asia pacific", "apac"))
|
|
add(*ANYWHERE, "Anywhere", None, ())
|
|
for code, names in _COUNTRY_LABELS.items():
|
|
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
|
|
for country, code, name in _subdivisions():
|
|
add("region", f"{country}-{code}", name, ("country", country), (fold(name),))
|
|
for region in REGIONS.values():
|
|
parent = ("country", region.country) if region.country else europe
|
|
add("region", region.key, region.label, parent,
|
|
(v for name in (region.key, region.label, *region.names)
|
|
for v in _variants(fold(name))))
|
|
for name in sorted(_CITY_REGIONS):
|
|
add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,))
|
|
|
|
# Towns, one per group, in the region that lists them first.
|
|
town_region: dict[str, str] = {}
|
|
for region in REGIONS.values():
|
|
for name in region.towns:
|
|
town_region.setdefault(_group(fold(name)), region.key)
|
|
folds: dict[str, dict[str, None]] = {}
|
|
shown: dict[str, str] = {}
|
|
|
|
def add_town(folded: str, name: str) -> None:
|
|
key = _group(folded)
|
|
folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded)))
|
|
shown.setdefault(key, name)
|
|
|
|
for spellings in _ALIAS_GROUPS:
|
|
for spelling in spellings:
|
|
add_town(fold(spelling), spellings[0])
|
|
for region in REGIONS.values():
|
|
for name in region.towns:
|
|
add_town(fold(name), name)
|
|
for city in _CITIES:
|
|
if city not in _CITY_REGIONS and city not in _REGION_BY_NAME:
|
|
add_town(city, _TOWN_LABELS.get(city) or city.title())
|
|
for key, forms in folds.items():
|
|
country = next((_CITIES[f] for f in forms if f in _CITIES), "")
|
|
if key in town_region:
|
|
parent = ("region", town_region[key])
|
|
elif country:
|
|
parent = ("country", country)
|
|
else:
|
|
raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES")
|
|
add("city", key, shown[key], parent, forms)
|
|
|
|
includes = [(("region", region.key), ("region", part))
|
|
for region in REGIONS.values() for part in region.includes]
|
|
return list(seeds.values()), includes
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Place:
|
|
"""A row of `places`."""
|
|
|
|
id: int
|
|
kind: str
|
|
key: str
|
|
name: str
|
|
parent_id: int | None
|
|
|
|
|
|
class Gazetteer:
|
|
"""The places, their aliases and what lies in what, as the database holds
|
|
them, read into memory: what resolves a role's place text to places and a
|
|
filter token to the places under it. Built by the store from its tables,
|
|
and rebuilt when they change."""
|
|
|
|
def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]],
|
|
aliases: Iterable[tuple[str, int]],
|
|
includes: Iterable[tuple[int, int]] = ()) -> None:
|
|
self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows}
|
|
self.by_key: dict[tuple[str, str], int] = {
|
|
(p.kind, p.key): p.id for p in self.places.values()}
|
|
by_fold: dict[str, list[int]] = {}
|
|
for folded, place_id in aliases:
|
|
by_fold.setdefault(folded, []).append(place_id)
|
|
self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()}
|
|
self._longest = max((len(f.split()) for f in self.aliases), default=1)
|
|
self._up: dict[int, set[int]] = {}
|
|
self._down: dict[int, set[int]] = {}
|
|
for p in self.places.values():
|
|
if p.parent_id is not None:
|
|
self._link(p.parent_id, p.id)
|
|
for whole, part in includes:
|
|
self._link(whole, part)
|
|
self._ancestors: dict[int, frozenset[int]] = {}
|
|
self._resolved: dict[str, tuple[int, ...]] = {}
|
|
self._overlap: dict[frozenset[int], frozenset[int]] = {}
|
|
self._names: dict[int, tuple[str, ...]] | None = None
|
|
self.anywhere: int | None = self.by_key.get(ANYWHERE)
|
|
|
|
def _link(self, whole: int, part: int) -> None:
|
|
self._up.setdefault(part, set()).add(whole)
|
|
self._down.setdefault(whole, set()).add(part)
|
|
|
|
def ancestors(self, place_id: int) -> frozenset[int]:
|
|
"""Every place `place_id` lies in, through parents and inclusions."""
|
|
if (found := self._ancestors.get(place_id)) is None:
|
|
seen: set[int] = set()
|
|
todo = list(self._up.get(place_id, ()))
|
|
while todo:
|
|
up = todo.pop()
|
|
if up not in seen:
|
|
seen.add(up)
|
|
todo += self._up.get(up, ())
|
|
found = self._ancestors[place_id] = frozenset(seen)
|
|
return found
|
|
|
|
def under(self, ids: Iterable[int]) -> frozenset[int]:
|
|
"""The places and every place that lies in one of them."""
|
|
seen: set[int] = set()
|
|
todo = list(ids)
|
|
while todo:
|
|
place_id = todo.pop()
|
|
if place_id not in seen:
|
|
seen.add(place_id)
|
|
todo += self._down.get(place_id, ())
|
|
return frozenset(seen)
|
|
|
|
def overlap(self, ids: Iterable[int]) -> frozenset[int]:
|
|
"""Every place a remote area may hold to overlap these: the places,
|
|
what lies under them, every place one of those lies in, and Anywhere.
|
|
Denmark's holds Europe, and Øresund through the Capital Region, so a
|
|
role remote within Europe is open to someone in Denmark."""
|
|
key = frozenset(ids)
|
|
if (found := self._overlap.get(key)) is None:
|
|
under = self.under(key)
|
|
wider = set(under)
|
|
for place_id in under:
|
|
wider |= self.ancestors(place_id)
|
|
if self.anywhere is not None:
|
|
wider.add(self.anywhere)
|
|
found = self._overlap[key] = frozenset(wider)
|
|
return found
|
|
|
|
def names(self, place_id: int) -> tuple[str, ...]:
|
|
"""The folds a place is written as: its aliases and its name."""
|
|
if self._names is None:
|
|
by_id: dict[int, dict[str, None]] = {}
|
|
for folded, found in self.aliases.items():
|
|
for one in found:
|
|
by_id.setdefault(one, {})[folded] = None
|
|
for place in self.places.values():
|
|
by_id.setdefault(place.id, {})[fold(place.name)] = None
|
|
self._names = {place_id: tuple(folds) for place_id, folds in by_id.items()}
|
|
return self._names.get(place_id, ())
|
|
|
|
def token(self, place_id: int) -> str:
|
|
"""The filter token that names a place: `country:DK`, `country:EU`
|
|
for Europe, `region:skane` for a region of `REGIONS`, `anywhere` for
|
|
Anywhere, else `place:` and the name folded."""
|
|
place = self.places[place_id]
|
|
if place_id == self.anywhere:
|
|
return "anywhere"
|
|
if (place.kind, place.key) == ("continent", "europe"):
|
|
return "country:EU"
|
|
if place.kind == "country":
|
|
return f"country:{place.key}"
|
|
if place.kind == "region" and place.key in REGIONS:
|
|
return f"region:{place.key}"
|
|
return f"place:{fold(place.name)}"
|
|
|
|
def remote_places(self, text: str, marked: bool = False) -> tuple[int, ...]:
|
|
"""The places the parts of a place text that say remote
|
|
(`remote_part`) name, where a remote hire may live: `Germany
|
|
(Remote)` is Germany, `Cardiff, London or Remote (UK)` the United
|
|
Kingdom, `Home Based - Worldwide` and `Remote, Global` Anywhere.
|
|
`marked` takes every part, for a name the model read as where a
|
|
remote hire may live ("Europe", "anywhere").
|
|
`global` and `international` are Anywhere only here, so `Global HQ,
|
|
Berlin` stays Berlin; beside another place Anywhere is dropped, so
|
|
`Anywhere in France, Belgium` is the two countries. `Remote` alone
|
|
names none."""
|
|
found: list[int] = []
|
|
for part in _PART.split(_AND_HOME.sub("", text or "")):
|
|
if not (marked or remote_part(part)):
|
|
continue
|
|
ids = [p for p in self.resolve(part) if p != self.anywhere]
|
|
if self.anywhere is not None and any(
|
|
f" {word} " in f" {fold(part)} " for word in _ANYWHERE_IF_REMOTE):
|
|
ids.append(self.anywhere)
|
|
if len(ids) > 1 and self.anywhere in ids:
|
|
ids.remove(self.anywhere)
|
|
found += ids
|
|
return tuple(dict.fromkeys(found))
|
|
|
|
def area_label(self, ids: Sequence[int]) -> str:
|
|
"""A remote area as a row shows it: the names in order, Anywhere as
|
|
"anywhere", "United States, Canada". More than three places are
|
|
shown as the countries they lie in, so twenty German states read
|
|
"Germany"."""
|
|
found = [p for p in ids if p in self.places]
|
|
if len(found) > 3:
|
|
found = list(dict.fromkeys(
|
|
self.by_key.get(("country", self.country(p)), p) for p in found))
|
|
return ", ".join("anywhere" if p == self.anywhere else self.places[p].name
|
|
for p in found)
|
|
|
|
def country(self, place_id: int) -> str:
|
|
"""The ISO code of the country a place is in, '' above or outside one."""
|
|
place: Place | None = self.places.get(place_id)
|
|
while place is not None and place.kind != "country":
|
|
place = self.places.get(place.parent_id) if place.parent_id is not None else None
|
|
return place.key if place is not None else ""
|
|
|
|
def codes(self, ids: Iterable[int]) -> frozenset[str]:
|
|
"""The ISO codes of the countries among `ids`, and `EU` with Europe:
|
|
what a `roles.country` column holds for a role of one of them."""
|
|
found = set()
|
|
for place_id in ids:
|
|
place = self.places[place_id]
|
|
if place.kind == "country":
|
|
found.add(place.key)
|
|
elif (place.kind, place.key) == ("continent", "europe"):
|
|
found.add("EU")
|
|
return frozenset(found)
|
|
|
|
def _specific(self, place_id: int) -> int:
|
|
return KINDS.index(self.places[place_id].kind)
|
|
|
|
def _hits(self, folded: str) -> list[tuple[int, ...]]:
|
|
words = folded.split()
|
|
hits = []
|
|
i = 0
|
|
while i < len(words):
|
|
for n in range(min(self._longest, len(words) - i), 0, -1):
|
|
if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None:
|
|
hits.append(ids)
|
|
i += n
|
|
break
|
|
else:
|
|
i += 1
|
|
return hits
|
|
|
|
def resolve(self, text: str) -> tuple[int, ...]:
|
|
"""The places one place text names, most specific first from the left:
|
|
`Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund`
|
|
is Malmö and Lund, `Ellwangen, Germany` is Germany, `Remote` is none.
|
|
|
|
Every alias the text holds as whole words is a candidate, longest
|
|
first. A fold that names several places is the one in the country
|
|
`country_of` reads off the text, if one is, and else the most specific
|
|
(`Luxembourg` the town before the country). A candidate that another
|
|
one lies in is dropped, and so is one outside the one country the text
|
|
names, or, naming none, outside the regions it names in one country:
|
|
`London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the
|
|
Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several
|
|
countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country
|
|
`country_of` reads (from a state code, `Paris, TX`, or a code in
|
|
parentheses, `Remote (DE)`), the text is that country."""
|
|
folded = fold(text)
|
|
if (found := self._resolved.get(folded)) is None:
|
|
parts = _SLASH.split(_AND_HOME.sub("", text or ""))
|
|
found = tuple(dict.fromkeys(
|
|
p for part in parts for sub in _PART.split(part) for p in self._resolve(sub)))
|
|
self._resolved[folded] = found
|
|
return found
|
|
|
|
def _resolve(self, text: str) -> list[int]:
|
|
"""`resolve` for one part of a text, cut where `or`, `/` or `;` end one
|
|
place."""
|
|
code = country_of(text)
|
|
country = self.by_key.get(("country", code)) if code and code != "EU" else None
|
|
within = self.under((country,)) if country is not None else frozenset()
|
|
picked: list[int] = []
|
|
for hit in self._hits(fold(text)):
|
|
choice = [p for p in hit if p in within] or list(hit)
|
|
best = max(choice, key=self._specific)
|
|
if best not in picked:
|
|
picked.append(best)
|
|
# The widest units named hold the rest: one country, else the regions
|
|
# of one country. Several countries are a list, not a path.
|
|
for kind in ("country", "region"):
|
|
named = [p for p in picked if self.places[p].kind == kind]
|
|
if len({self.country(p) for p in named}) == 1:
|
|
holds = self.under(named)
|
|
picked = [p for p in picked if p in holds or self.places[p].kind == "continent"
|
|
or any(p in self.ancestors(q) for q in named)]
|
|
if named:
|
|
break
|
|
wider = {a for p in picked for a in self.ancestors(p)}
|
|
picked = [p for p in picked if p not in wider]
|
|
if country is not None and not any(p in within or p in self.ancestors(country)
|
|
for p in picked):
|
|
picked = [country]
|
|
return picked
|
|
|
|
def lookup(self, token: str) -> frozenset[int] | None:
|
|
"""The places a filter token names, before what lies in them: `country:DK`
|
|
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
|
|
every place written so (a town, or a region or a country by name).
|
|
A `remote:` token names the places of its place token, `remote:anywhere`
|
|
Anywhere, `remote:any` none. None when the gazetteer knows no such
|
|
place."""
|
|
tok = parse(token)
|
|
if tok.kind == "remote":
|
|
if tok.value != "anywhere":
|
|
return self.lookup(tok.value) if tok.value not in ("", "any") else None
|
|
found = {self.anywhere} if self.anywhere is not None else set()
|
|
elif tok.kind == "country":
|
|
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
|
|
found = {self.by_key[key]} if key in self.by_key else set()
|
|
elif tok.kind == "region":
|
|
found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \
|
|
else set(self.aliases.get(tok.value, ()))
|
|
else:
|
|
found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())}
|
|
return frozenset(found) or None
|