Files
jobboard/app/places.py
T
WolfandClaude Opus 5.5 5410131f4a Link DEU, Deutschlandweit, the German remote regions, AMER and North America
DEU and Deutschlandweit are spellings of Germany, AMER and North America of
the United States. Seevetal and Neukirchen-Vluyn are German towns, and
Niedersachsen, Nord-, Ost- and Süddeutschland are regions of Germany named
without towns. Fleet Place Office and Solaris SE stay unlinked.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 13:22:33 +02:00

990 lines
43 KiB
Python

"""Places as a filter: what a board writes in `place`, matched by whole words.
A location line is free text: "København K", "Kgs. Lyngby", "Aarhus N, Region
Central Jutland, Denmark". A substring match finds `Lund` in `Billund` and
`Denmark` in nothing that only names a town. Here a place is folded (casefolded,
marks stripped, `ø` to `o`, `æ` to `ae`, `å` to `a`, punctuation to spaces) and
matched as whole words, so `Lund` is `lund` and no other word. A filter is one
of three tokens, and several tokens are joined by OR:
country:DK the role's country, or a line that names it while `country`
is empty or `EU` (a board that lists several offices)
region:skane any town or name of a region in `REGIONS`
place:malmo the town's fold and its aliases, so `copenhagen`,
`kobenhavn` and `kopenhamn` find each other
`matches` answers whether one role's folded place and country fit a token, in
Python: a region is over a hundred phrases, and as `LIKE` clauses that costs
several hundred milliseconds a query, as a set intersection on the words of a
line it costs a few. `register` gives a connection that answer as the SQL
function `place_match`, and `match_sql` returns the clause that calls it.
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
chip's text back from the address.
The filter and the gates no longer match words: a role's places are resolved
against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
and a token is the places under the place it names. `matches` stays the rule
for a `place:` token the gazetteer does not know. This module opens no
database; the caller hands over the places it holds, or its connection.
"""
from __future__ import annotations
import re
import sqlite3
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass
from functools import lru_cache
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
country_of)
__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed",
"Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse",
"placeless", "register", "seed", "suggest", "towns"]
def fold(text: str) -> str:
"""`Skåne län` -> `skane lan`, `København K` -> `kobenhavn k`, `Kgs. Lyngby`
-> `kgs lyngby`. What `roles.place_fold` holds. `å` needs no rule of its own:
NFKD splits it into `a` and a ring, and the ring is a mark."""
return _place_fold(text)
# ------------------------------------------------------------------ countries
def _countries() -> dict[str, tuple[str, ...]]:
"""Every code to the names `_COUNTRY_NAMES` gives it, English name first."""
found: dict[str, tuple[str, ...]] = {}
for line in _COUNTRY_NAMES.split("\n"):
if line:
code, rest = line.split(" ", 1)
found[code] = tuple(name.strip() for name in rest.split("|"))
return found
_COUNTRY_LABELS = _countries()
#: A code to the folded names a country goes by.
_COUNTRY_FOLDS: dict[str, tuple[str, ...]] = {
code: tuple(dict.fromkeys(filter(None, map(fold, names))))
for code, names in _COUNTRY_LABELS.items()}
#: What a line has to say for a role of that country whose `country` column is
#: empty or `EU`: a name of the country, or a city `country_of` knows in it. A
#: board that lists offices ("Copenhagen; London; Warsaw") names cities only.
_COUNTRY_MENTIONS: dict[str, tuple[str, ...]] = {
code: tuple(dict.fromkeys((*names, *(fold(c) for c, cc in _CITIES.items() if cc == code))))
for code, names in _COUNTRY_FOLDS.items()}
#: A folded country name to its code.
_COUNTRY_BY_NAME: dict[str, str] = {
name: code for code, names in _COUNTRY_FOLDS.items() for name in names}
#: Words that are not a town: a country, a macro region, how the role is worked.
_NOT_A_TOWN = frozenset(_COUNTRY_BY_NAME) | frozenset(map(fold, _REGIONS)) \
| {"remote", "hybrid"}
# -------------------------------------------------------------------- aliases
#: Towns boards write in more than one language, first spelling shown. Each
#: group is symmetric: any spelling finds the others. The Nordics and the DACH
#: countries in full for their large cities, and the European cities that
#: appear both ways in boards.
_ALIAS_GROUPS: tuple[tuple[str, ...], ...] = (
# Denmark
("Copenhagen", "København", "Kobenhavn", "Köpenhamn", "Kopenhamn", "Kopenhagen",
"Copenhague", "Kopenhaga"),
("Aarhus", "Århus"),
("Aalborg", "Ålborg"),
("Kgs. Lyngby", "Kongens Lyngby", "Lyngby"),
("Taastrup", "Tåstrup"),
("Elsinore", "Helsingør"),
# Sweden
("Gothenburg", "Göteborg", "Goteborg"),
("Stockholm",),
("Malmö", "Malmo"),
("Helsingborg", "Hälsingborg"),
("Västerås", "Vasteras"),
("Linköping", "Linkoping"),
("Örebro", "Orebro"),
("Norrköping", "Norrkoping"),
("Jönköping", "Jonkoping"),
# Norway, Finland, Iceland
("Oslo",),
("Bergen",),
("Trondheim", "Trondhjem"),
("Helsinki", "Helsingfors"),
("Espoo", "Esbo"),
("Vantaa", "Vanda"),
("Reykjavik", "Reykjavík"),
# Germany
("Munich", "München", "Muenchen", "Munchen"),
("Cologne", "Köln", "Koeln", "Koln"),
("Nuremberg", "Nürnberg", "Nuernberg", "Nurnberg"),
("Düsseldorf", "Duesseldorf", "Dusseldorf"),
("Hanover", "Hannover"),
("Mainz", "Mayence"),
("Frankfurt", "Frankfurt am Main"),
("Göttingen", "Goettingen", "Gottingen"),
("Würzburg", "Wuerzburg", "Wurzburg"),
("Tübingen", "Tuebingen", "Tubingen"),
("Lübeck", "Luebeck", "Lubeck"),
("Münster", "Muenster", "Munster"),
("Saarbrücken", "Saarbruecken", "Saarbrucken"),
("Neukirchen-Vluyn",),
# Austria
("Vienna", "Wien", "Vienne"),
("Salzburg",),
("Innsbruck",),
# Switzerland
("Zurich", "Zürich", "Zuerich"),
("Geneva", "Genève", "Geneve", "Genf", "Ginevra"),
("Basel", "Basle", "Bâle"),
("Bern", "Berne"),
("Lucerne", "Luzern"),
("St. Gallen", "Sankt Gallen", "Saint Gall"),
# The rest of Europe
("Prague", "Praha", "Prag"),
("Warsaw", "Warszawa", "Warschau"),
("Krakow", "Kraków", "Cracow"),
("Wroclaw", "Wrocław", "Breslau"),
("Gdansk", "Gdańsk", "Danzig"),
("Poznan", "Poznań", "Posen"),
("Lisbon", "Lisboa", "Lissabon"),
("Milan", "Milano", "Mailand"),
("Rome", "Roma", "Rom"),
("Turin", "Torino"),
("Florence", "Firenze"),
("Naples", "Napoli", "Neapel"),
("Venice", "Venezia", "Venedig"),
("Genoa", "Genova"),
("Brussels", "Bruxelles", "Brussel", "Brüssel"),
("Antwerp", "Antwerpen", "Anvers"),
("Ghent", "Gent", "Gand"),
("Bruges", "Brugge"),
("Liège", "Liege", "Luik", "Lüttich"),
("The Hague", "Den Haag", "s-Gravenhage"),
("Luxembourg", "Luxemburg"),
("Seville", "Sevilla"),
("Athens", "Athina", "Athen"),
("Bucharest", "București", "Bucuresti"),
("Belgrade", "Beograd"),
("Bratislava", "Pressburg"),
("Kyiv", "Kiev"),
("Cluj-Napoca", "Cluj"),
("Timișoara", "Timisoara"),
("Marseille", "Marseilles"),
("Bolzano", "Bozen"),
)
def _variants(text: str) -> tuple[str, ...]:
"""A fold and its `aa` spelling: Danish writes `å` as `aa` (`Aarhus`, `Taastrup`)
and `Århus` and `Tåstrup` fold to `arhus` and `tastrup`."""
return tuple(dict.fromkeys((text, text.replace("aa", "a"))))
def _aliases() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]:
"""Every folded spelling to its whole group (all folded), and to its shown
spelling. The group starts with the first spelling listed."""
groups: dict[str, tuple[str, ...]] = {}
shown: dict[str, str] = {}
for group in _ALIAS_GROUPS:
members = tuple(dict.fromkeys(v for name in group for v in _variants(fold(name))))
for name in group:
shown.setdefault(fold(name), name)
for member in members:
groups[member] = members
return groups, shown
_GROUPS, _SHOWN = _aliases()
def _expand(text: str) -> tuple[str, ...]:
"""A folded town and everything it is also written as."""
return _GROUPS.get(text) or _variants(text)
# -------------------------------------------------------------------- regions
@dataclass(frozen=True)
class Region:
"""A region below a country, or across two. `names` are what a board calls
the region itself, `towns` the places in it; a line is in the region when it
has one of either as whole words, or is in a region it `includes`, by key.
Only `names` are what the region is suggested and read as. Spellings are
shown with their marks and folded when used."""
key: str
label: str
names: tuple[str, ...]
towns: tuple[str, ...]
country: str = ""
includes: tuple[str, ...] = ()
def _split(text: str) -> tuple[str, ...]:
return tuple(part.strip() for part in text.split(",") if part.strip())
_SKANE = Region(
"skane", "Skåne",
_split("Skåne, Skåne län, Skåne County, Region Skåne, Skåne Region, Scania, Skaane"),
_split("""Malmö, Lund, Helsingborg, Kristianstad, Landskrona, Trelleborg, Ängelholm,
Hässleholm, Ystad, Eslöv, Höganäs, Staffanstorp, Lomma, Bjuv, Burlöv, Kävlinge,
Svedala, Vellinge, Simrishamn, Tomelilla, Sjöbo, Svalöv, Klippan, Åstorp,
Bromölla, Osby, Perstorp, Örkelljunga, Östra Göinge, Hörby, Höör, Båstad,
Skurup, Arlöv, Limhamn, Dalby, Åkarp, Bjärred, Höllviken, Ljungbyhed,
Löddeköpinge, Furulund, Genarp, Veberöd, Skanör, Falsterbo, Mölle, Arild"""),
"SE")
_HOVEDSTADEN = Region(
"hovedstaden", "Capital Region of Denmark",
_split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"),
_split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup,
Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte,
Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre,
Høje-Taastrup, Hørsholm, Ishøj, Lyngby-Taarbæk, Rudersdal, Rødovre, Tårnby,
Vallensbæk, Kgs. Lyngby, Kongens Lyngby, Lyngby, Virum, Holte, Nærum, Birkerød,
Vedbæk, Skodsborg, Klampenborg, Charlottenlund, Hellerup, Ordrup, Bagsværd,
Søborg, Mørkhøj, Måløv, Skovlunde, Smørum, Ølstykke, Stenløse, Værløse, Farum,
Slangerup, Hedehusene, Taastrup, Kastrup, Amager, Valby, Vanløse, Brønshøj,
Nørrebro, Østerbro, Vesterbro, Nordhavn, Ørestad, Sydhavn, Islands Brygge,
Refshaleøen, Humlebæk, Kokkedal, Espergærde, Snekkersten, Rungsted, Rungsted Kyst,
Lillerød, Frederiksværk, Hundested, Helsinge, Gilleleje, Græsted, Vallensbæk Strand,
Skævinge, Rønne, Nexø, Allinge, Svaneke, Aakirkeby, Hasle"""),
"DK")
# Region Sjælland, not the island: Copenhagen is on Zealand too, but a board that
# writes "Region Zealand" means the region and the Capital Region has its own key.
# The bare word is left out of `names` since "New Zealand" would match it.
_ZEALAND = Region(
"zealand", "Zealand",
_split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"),
_split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved,
Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg,
Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing,
Haslev, Korsør, Skælskør, Præstø, Stege, Fakse, Rødby, Rødbyhavn, Jyllinge,
Viby Sjælland, Solrød Strand, Fuglebjerg, Gørlev, Dianalund, Jyderup, Hvalsø,
Tølløse, Hørve, Fårevejle, Bjæverskov, Nørre Asmindrup"""),
"DK")
_JUTLAND = Region(
"jutland", "Jutland",
_split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland,
North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland,
East Jutland, West Jutland, Vestjylland, Østjylland"""),
_split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning,
Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov,
Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå,
Hjørring, Frederikshavn, Skagen, Thisted, Struer, Ikast, Brande, Varde, Ribe,
Tønder, Vojens, Hadsten, Hobro, Skanderborg, Odder, Hedensted, Vejen, Ebeltoft,
Lemvig, Rønde, Egå, Lystrup, Hinnerup, Tilst, Hammel, Galten, Sabro, Harlev,
Løsning, Højbjerg, Juelsminde, Nørresundby, Brønderslev, Aars, Nibe, Sæby, Løgstør, Tarm,
Skjern, Padborg, Rødekro, Toftlund, Bramming, Kruså, Brabrand, Ry, Gjern"""),
"DK")
_FUNEN = Region(
"funen", "Funen",
_split("Funen, Fyn, Region Fyn"),
_split("""Odense, Svendborg, Nyborg, Faaborg, Fåborg, Middelfart, Assens, Kerteminde,
Bogense, Langeskov, Ringe, Otterup, Munkebo, Vissenbjerg, Årslev, Tommerup,
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
"DK")
# The Øresund region is the two capital regions either side of the sound. It is
# suggested on its own names only; a line in either region is in it.
_ORESUND = Region(
"oresund", "Øresund",
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"),
(), includes=("hovedstaden", "skane"))
# Germany's by the names boards write for a remote role's area, towns not
# listed: a town in Germany is found through the country.
_GERMANY = tuple(Region(key, label, _split(names), (), "DE") for key, label, names in (
("niedersachsen", "Niedersachsen", "Niedersachsen, Lower Saxony"),
("norddeutschland", "Norddeutschland", "Norddeutschland, Northern Germany"),
("ostdeutschland", "Ostdeutschland", "Ostdeutschland, Eastern Germany"),
("suddeutschland", "Süddeutschland", "Süddeutschland, Southern Germany")))
REGIONS: dict[str, Region] = {
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN, *_GERMANY)}
def _phrases(region: Region) -> tuple[str, ...]:
"""Every folded phrase that puts a line in the region, with the other
spellings of its towns, and those of the regions it includes."""
found: dict[str, None] = {}
for text in (*region.names, *region.towns):
for form in _expand(fold(text)):
found[form] = None
for key in region.includes:
found.update(dict.fromkeys(_phrases(REGIONS[key])))
return tuple(found)
_PHRASES: dict[str, tuple[str, ...]] = {key: _phrases(r) for key, r in REGIONS.items()}
#: A folded key, label or name to the region key; a name in two regions ("Capital
#: Region of Denmark" is also in Øresund) belongs to the one that lists it first.
_REGION_BY_NAME: dict[str, str] = {}
for _key, _region in REGIONS.items():
for _name in (_region.key, _region.label, *_region.names):
_REGION_BY_NAME.setdefault(fold(_name), _key)
#: What a place is shown as when a chip is read from the address: every alias
#: spelling, every region and every town of one, as written above.
_TOWN_LABELS: dict[str, str] = dict(_SHOWN)
for _region in REGIONS.values():
for _town in (_region.label, *_region.towns, *_region.names):
_TOWN_LABELS.setdefault(fold(_town), _town)
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
def _group(folded: str) -> str:
"""The key of the town a fold is: the first spelling of its alias group, else
the fold with `aa` as `a`."""
return _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
def _known_towns() -> dict[str, str]:
"""Every town the tables name, as a fold, to its group: the alias groups, the
towns of the regions and the cities `country_of` knows."""
names = [*_GROUPS, *(fold(city) for city in _CITIES),
*(fold(town) for region in REGIONS.values() for town in region.towns)]
return {form: _group(form) for name in names for form in _variants(name)}
_KNOWN = _known_towns()
_KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN)
# --------------------------------------------------------------------- tokens
@dataclass(frozen=True)
class Token:
"""One place filter. `value` is a country code, a region key or a fold."""
kind: str
value: str
label: str
def __str__(self) -> str:
return f"{self.kind}:{self.value}"
def _region_key(text: str) -> str:
folded = fold(text)
return _REGION_BY_NAME.get(folded, folded)
def parse(token: str) -> Token:
"""`country:DK`, `region:skane` or `place:malmo`. Bare text, or a prefix that
is none of these, is a place: `Skåne` is `place:skane`. A country by name
(`country:Denmark`) and a region by label (`region:Skåne`) are read as well."""
kind, sep, value = (token or "").partition(":")
kind = kind.strip().lower()
if not sep or kind not in ("country", "region", "place"):
kind, value = "place", token or ""
value = value.strip()
if kind == "country":
code = value.upper()
if code not in _COUNTRY_LABELS:
code = _COUNTRY_BY_NAME.get(fold(value), code)
return Token("country", code, _COUNTRY_LABELS.get(code, (code,))[0])
if kind == "region":
key = _region_key(value)
return Token("region", key, REGIONS[key].label if key in REGIONS else key)
folded = fold(value)
return Token("place", folded, _TOWN_LABELS.get(folded) or folded.title())
def label(token: str | Token) -> str:
"""The text of a chip read from the address: `country:DK` is `Denmark`,
`region:skane` is `Skåne`, `place:malmo` is `Malmö`. A town the tables do not
know is its fold in title case."""
return (token if isinstance(token, Token) else parse(token)).label
# ------------------------------------------------------------------- matching
SQL_FUNCTION = "place_match"
def known(token: str) -> bool:
"""Whether the token names something: a country of the table (or `EU`), a
region of `REGIONS`, a place with a value. `region:atlantis`, `country:ZZ`
and `place:` are not."""
return _known(parse(token))
def _known(tok: Token) -> bool:
if tok.kind == "country":
return tok.value in _COUNTRY_LABELS or tok.value == "EU"
if tok.kind == "region":
return tok.value in REGIONS
return bool(tok.value)
@dataclass(frozen=True)
class _Matcher:
"""What a token asks of a role. `phrases` are whole-word phrases of the
folded place; `countries` are `(code, mentions)`: the role's own country, or
a line with one of the mentions while `country` is empty or `EU`, since a
board listing several offices has no single country."""
phrases: frozenset[str]
countries: tuple[tuple[str, frozenset[str]], ...]
longest: int
@lru_cache(maxsize=1024)
def _matcher(token: str) -> _Matcher | None:
"""The token read once: `None` when it names nothing.
A `place:` value that is also a region or a country is matched as that too:
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
written as a bare word means what the owner meant by it."""
tok = parse(token)
if not _known(tok):
return None
phrases: set[str] = set()
countries: list[tuple[str, frozenset[str]]] = []
if tok.kind == "country":
countries.append((tok.value, frozenset(_COUNTRY_MENTIONS.get(tok.value, ()))))
elif tok.kind == "region":
phrases.update(_PHRASES[tok.value])
else:
phrases.update(_expand(tok.value))
phrases.update(_PHRASES.get(_REGION_BY_NAME.get(tok.value, ""), ()))
if code := _COUNTRY_BY_NAME.get(tok.value):
countries.append((code, frozenset(_COUNTRY_MENTIONS.get(code, ()))))
phrases.discard("")
longest = max((len(p.split()) for p in (*phrases, *(m for _, ms in countries for m in ms))),
default=1)
return _Matcher(frozenset(phrases), tuple(countries), longest)
@lru_cache(maxsize=32768)
def _runs(place_fold: str, longest: int) -> frozenset[str]:
"""Every run of up to `longest` words of a folded line, each a whole-word
phrase the line holds."""
words = place_fold.split()
return frozenset(" ".join(words[i:i + n])
for n in range(1, longest + 1) for i in range(len(words) - n + 1))
def matches(place_fold: str, country: str, token: str) -> bool:
"""Whether a role whose folded place and country are these is found by the
token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or
a Danish city) while `country` is empty or `EU`. `region:skane` is any of
its names or towns as whole words, `place:malmo` the town and its aliases.
A token that names nothing (`known` is false) matches no role."""
match = _matcher(token)
if match is None:
return False
runs = None
if match.phrases:
runs = _runs(place_fold, match.longest)
if not match.phrases.isdisjoint(runs):
return True
for code, mentions in match.countries:
if country == code:
return True
if mentions and country in ("", "EU"):
runs = _runs(place_fold, match.longest) if runs is None else runs
if not mentions.isdisjoint(runs):
return True
return False
def _place_match(place_fold: str | None, country: str | None, token: str) -> int:
return int(matches(place_fold or "", country or "", token))
def register(conn: sqlite3.Connection) -> None:
"""Give a connection `place_match(place_fold, country, token)`, which is 1 when
`matches` is true. `match_sql` returns a clause that calls it."""
conn.create_function(SQL_FUNCTION, 3, _place_match, deterministic=True)
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
-> tuple[str, list]:
"""SQL over the two columns for one token, with one `?` parameter. It calls
`place_match`, so the connection needs `register`. Several tokens are joined
with OR by the caller. Bare text is a place. A token that names nothing
(`known` is false) is `0`, which matches no role."""
tok = parse(token)
if not _known(tok):
return "0", []
return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)]
# ---------------------------------------------------------------- suggestions
@dataclass(frozen=True)
class Suggestion:
"""`token` is what the chip holds, `kind` is `country`, `region` or `place`,
`count` the open roles it stands for."""
token: str
label: str
kind: str
count: int
def _prefixes(q: str, candidates: Iterable[str]) -> bool:
"""`q` starts a word of a candidate, or a run of words: `kgs ly` finds
`kgs lyngby`, `hague` finds `the hague`."""
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
#: Where one town ends in a line: `;` and `|` between places, and `or` / `eller` /
#: `og` between two towns in one ("København V eller Aalborg").
_PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Danish boards append "and the possibility of working from home".
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
#: A slash between two places, "Malmö/Lund", "Berlin / Munich".
_SLASH = re.compile(r"\s*/\s*")
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
#: A country or "Remote" ahead of the town, "Sweden - Lund", "Remote-New Zealand".
_LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$")
def _towns(place: str) -> list[str]:
"""The first comma part of each place of a line: `Aarhus N, Region Central
Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal
district is dropped, since `place:kobenhavn` finds `København K` anyway."""
found = []
for part in _PART.split(_AND_HOME.sub("", place or "")):
head = part.split(",")[0].strip()
if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN:
head = lead[2]
head = _DISTRICT.sub("", head)
if fold(head) and fold(head) not in _NOT_A_TOWN:
found.append(head)
return found
def _towns_in(folded: str) -> list[str]:
"""The known towns a fold holds as whole words, longest first from the left,
as their groups: `munich berlin` is Munich and Berlin, `copenhagen office` is
Copenhagen, `hoje taastrup` is one town and not also Taastrup."""
words = folded.split()
found: list[str] = []
i = 0
while i < len(words):
for n in range(min(_KNOWN_LONGEST, len(words) - i), 0, -1):
if (group := _KNOWN.get(" ".join(words[i:i + n]))) is not None:
found.append(group)
i += n
break
else:
i += 1
return list(dict.fromkeys(found))
def towns(place: str) -> frozenset[str]:
"""The towns a line names, each as its group's key, so `København K` and
`Copenhagen, Denmark` are both `{"copenhagen"}`: the first comma part of each
place, and every known town the line holds as whole words, so a list such as
`Aalborg, Copenhagen, Kolding` gives all three. A state or province after
the town is not a town, so `Mountain View, California` and `San Francisco,
California` share none. Empty for a line that names only a country, a region
or how the role is worked."""
found = {_group(fold(town)) for town in _towns(place)}
found.update(_towns_in(fold(place or "")))
return frozenset(found)
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
"""Open roles per region, over folded lines. A line is cut into every run of
words once, and each region asks whether it holds one of its phrases."""
sets = {key: frozenset(_PHRASES[key]) for key in keys}
longest = max((len(p.split()) for phrases in sets.values() for p in phrases), default=1)
counts: Counter[str] = Counter()
for text, roles in places:
runs = _runs(text, longest)
for key, phrases in sets.items():
if not phrases.isdisjoint(runs):
counts[key] += roles
return counts
def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
-> list[Suggestion]:
"""Tokens for the text typed so far. `places` is `(place, country, open
roles)`. Countries first, then regions, then towns, each by open roles.
A country is suggested when `q` starts a word of one of its names, with the
roles that carry its code. A region likewise on its label and names, with the
roles whose line is in it. A town is the first comma part of each `;` part of
a line, grouped by fold and by alias (`Copenhagen` and `København` are one
town, shown as its commonest spelling). Countries, regions and towns with no
open role are left out: a chip that finds nothing helps nobody."""
text = fold(q)
if not text:
return []
rows = [(place or "", country or "", int(roles)) for place, country, roles in places]
by_country: Counter[str] = Counter()
for _, country, roles in rows:
by_country[country] += roles
countries = [Suggestion(f"country:{code}", names[0], "country", by_country[code])
for code, names in _COUNTRY_LABELS.items()
if by_country[code] and _prefixes(text, _COUNTRY_FOLDS[code])]
named = [key for key, r in REGIONS.items()
if _prefixes(text, [fold(r.label), *map(fold, r.names)])]
counts = _region_counts(named, [(fold(place), roles) for place, _, roles in rows])
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
for key in named if counts[key]]
# A line's town is a group. A line that is a town is that group's spelling;
# one that only holds a known town ("Copenhagen Office", "Munich - Berlin")
# counts under each such town and shows as none; anything else is its own.
counts_by_town: Counter[str] = Counter()
spellings: dict[str, Counter[str]] = {}
written: dict[str, set[str]] = {}
for place, _, roles in rows:
for town in _towns(place):
folded = fold(town)
if folded not in _KNOWN and (inside := _towns_in(folded)):
for group in inside:
counts_by_town[group] += roles
continue
group = _group(folded)
counts_by_town[group] += roles
spellings.setdefault(group, Counter())[town] += roles
written.setdefault(group, set()).add(folded)
towns = []
for group, total in counts_by_town.items():
forms = {group, *written.get(group, ())}
if not _prefixes(text, [form for folded in forms for form in _expand(folded)]):
continue
seen = spellings.get(group)
shown = min(seen, key=lambda s: (-seen[s], s)) if seen \
else _TOWN_LABELS.get(group) or group.title()
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total))
countries.sort(key=lambda s: (-s.count, s.label))
regions.sort(key=lambda s: (-s.count, s.label))
towns.sort(key=lambda s: (-s.count, s.label))
return (countries + regions + towns)[:limit]
# ------------------------------------------------------------------ gazetteer
#: The kinds of place, widest first: the index is how specific a kind is.
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
#: Entries of `_CITIES` that name a region rather than a town.
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
"bay area"))
#: Words a place text holds when it names how or where-ever a role is worked
#: rather than a place: "Remote", "Multiple Locations", "Home based". A text
#: of these alone is no place, so it is never listed as unresolved.
_PLACELESS = frozenset((
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
def placeless(text: str) -> bool:
"""Whether a place text names no place at all: empty, or only numbers and
words such as `Remote`, `Hybrid`, `2 Locations`."""
return all(word in _PLACELESS or word.isdigit() for word in fold(text).split())
@dataclass(frozen=True)
class Seed:
"""One place as the code knows it: `key` is unique within `kind` (an ISO
code for a country, a region's key, a town's group), `parent` the
`(kind, key)` of the place it lies in, `aliases` the folds it is written
as."""
kind: str
key: str
name: str
parent: tuple[str, str] | None
aliases: tuple[str, ...]
def _subdivisions() -> list[tuple[str, str, str]]:
"""`(country, code, name)` of every US state and Canadian province."""
found = []
for line in _SUBDIVISIONS.split("\n"):
if line:
country, rest = line.split(" ", 1)
for entry in rest.split("|"):
name, code = entry.strip().rsplit(" ", 1)
found.append((country, code, name))
return found
def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
"""The gazetteer the dictionaries in code describe, parents before their
children, and the inclusions beyond the tree (Øresund holds the Capital
Region and Skåne, which lie in two countries).
Europe is a continent over the countries of `_EUROPE`, and is also what
`EU`, `EMEA` and `European Union` name. Asia Pacific (`APAC`) is a continent over
nothing. Every country of `_COUNTRY_NAMES`
by its names; the US states and Canadian provinces as regions; the
regions of `REGIONS`; the towns of the alias groups, of the regions and
of `_CITIES`, one place per group (`Copenhagen`, `København`,
`Kopenhamn` are one), in their region when a region lists them and else
in their country."""
europe = ("continent", "europe")
seeds: dict[tuple[str, str], Seed] = {}
def add(kind: str, key: str, name: str, parent: tuple[str, str] | None,
aliases: Iterable[str]) -> None:
folds = tuple(dict.fromkeys(a for a in aliases if a))
if (kind, key) in seeds:
old = seeds[(kind, key)]
folds = tuple(dict.fromkeys((*old.aliases, *folds)))
seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds)
else:
seeds[(kind, key)] = Seed(kind, key, name, parent, folds)
add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS)))
# Asia Pacific holds no country: a role placed there alone keeps the
# host's country, and one beside a country is that country.
add("continent", "asia-pacific", "Asia Pacific", None, ("asia pacific", "apac"))
for code, names in _COUNTRY_LABELS.items():
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
for country, code, name in _subdivisions():
add("region", f"{country}-{code}", name, ("country", country), (fold(name),))
for region in REGIONS.values():
parent = ("country", region.country) if region.country else europe
add("region", region.key, region.label, parent,
(v for name in (region.key, region.label, *region.names)
for v in _variants(fold(name))))
for name in sorted(_CITY_REGIONS):
add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,))
# Towns, one per group, in the region that lists them first.
town_region: dict[str, str] = {}
for region in REGIONS.values():
for name in region.towns:
town_region.setdefault(_group(fold(name)), region.key)
folds: dict[str, dict[str, None]] = {}
shown: dict[str, str] = {}
def add_town(folded: str, name: str) -> None:
key = _group(folded)
folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded)))
shown.setdefault(key, name)
for spellings in _ALIAS_GROUPS:
for spelling in spellings:
add_town(fold(spelling), spellings[0])
for region in REGIONS.values():
for name in region.towns:
add_town(fold(name), name)
for city in _CITIES:
if city not in _CITY_REGIONS and city not in _REGION_BY_NAME:
add_town(city, _TOWN_LABELS.get(city) or city.title())
for key, forms in folds.items():
country = next((_CITIES[f] for f in forms if f in _CITIES), "")
if key in town_region:
parent = ("region", town_region[key])
elif country:
parent = ("country", country)
else:
raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES")
add("city", key, shown[key], parent, forms)
includes = [(("region", region.key), ("region", part))
for region in REGIONS.values() for part in region.includes]
return list(seeds.values()), includes
@dataclass(frozen=True)
class Place:
"""A row of `places`."""
id: int
kind: str
key: str
name: str
parent_id: int | None
class Gazetteer:
"""The places, their aliases and what lies in what, as the database holds
them, read into memory: what resolves a role's place text to places and a
filter token to the places under it. Built by the store from its tables,
and rebuilt when they change."""
def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]],
aliases: Iterable[tuple[str, int]],
includes: Iterable[tuple[int, int]] = ()) -> None:
self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows}
self.by_key: dict[tuple[str, str], int] = {
(p.kind, p.key): p.id for p in self.places.values()}
by_fold: dict[str, list[int]] = {}
for folded, place_id in aliases:
by_fold.setdefault(folded, []).append(place_id)
self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()}
self._longest = max((len(f.split()) for f in self.aliases), default=1)
self._up: dict[int, set[int]] = {}
self._down: dict[int, set[int]] = {}
for p in self.places.values():
if p.parent_id is not None:
self._link(p.parent_id, p.id)
for whole, part in includes:
self._link(whole, part)
self._ancestors: dict[int, frozenset[int]] = {}
self._resolved: dict[str, tuple[int, ...]] = {}
def _link(self, whole: int, part: int) -> None:
self._up.setdefault(part, set()).add(whole)
self._down.setdefault(whole, set()).add(part)
def ancestors(self, place_id: int) -> frozenset[int]:
"""Every place `place_id` lies in, through parents and inclusions."""
if (found := self._ancestors.get(place_id)) is None:
seen: set[int] = set()
todo = list(self._up.get(place_id, ()))
while todo:
up = todo.pop()
if up not in seen:
seen.add(up)
todo += self._up.get(up, ())
found = self._ancestors[place_id] = frozenset(seen)
return found
def under(self, ids: Iterable[int]) -> frozenset[int]:
"""The places and every place that lies in one of them."""
seen: set[int] = set()
todo = list(ids)
while todo:
place_id = todo.pop()
if place_id not in seen:
seen.add(place_id)
todo += self._down.get(place_id, ())
return frozenset(seen)
def country(self, place_id: int) -> str:
"""The ISO code of the country a place is in, '' above or outside one."""
place: Place | None = self.places.get(place_id)
while place is not None and place.kind != "country":
place = self.places.get(place.parent_id) if place.parent_id is not None else None
return place.key if place is not None else ""
def codes(self, ids: Iterable[int]) -> frozenset[str]:
"""The ISO codes of the countries among `ids`, and `EU` with Europe:
what a `roles.country` column holds for a role of one of them."""
found = set()
for place_id in ids:
place = self.places[place_id]
if place.kind == "country":
found.add(place.key)
elif (place.kind, place.key) == ("continent", "europe"):
found.add("EU")
return frozenset(found)
def _specific(self, place_id: int) -> int:
return KINDS.index(self.places[place_id].kind)
def _hits(self, folded: str) -> list[tuple[int, ...]]:
words = folded.split()
hits = []
i = 0
while i < len(words):
for n in range(min(self._longest, len(words) - i), 0, -1):
if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None:
hits.append(ids)
i += n
break
else:
i += 1
return hits
def resolve(self, text: str) -> tuple[int, ...]:
"""The places one place text names, most specific first from the left:
`Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund`
is Malmö and Lund, `Paderborn, Germany` is Germany, `Remote` is none.
Every alias the text holds as whole words is a candidate, longest
first. A fold that names several places is the one in the country
`country_of` reads off the text, if one is, and else the most specific
(`Luxembourg` the town before the country). A candidate that another
one lies in is dropped, and so is one outside the one country the text
names, or, naming none, outside the regions it names in one country:
`London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the
Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several
countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country
`country_of` reads (from a state code, `Paris, TX`, or a code in
parentheses, `Remote (DE)`), the text is that country."""
folded = fold(text)
if (found := self._resolved.get(folded)) is None:
parts = _SLASH.split(_AND_HOME.sub("", text or ""))
found = tuple(dict.fromkeys(
p for part in parts for sub in _PART.split(part) for p in self._resolve(sub)))
self._resolved[folded] = found
return found
def _resolve(self, text: str) -> list[int]:
"""`resolve` for one part of a text, cut where `or`, `/` or `;` end one
place."""
code = country_of(text)
country = self.by_key.get(("country", code)) if code and code != "EU" else None
within = self.under((country,)) if country is not None else frozenset()
picked: list[int] = []
for hit in self._hits(fold(text)):
choice = [p for p in hit if p in within] or list(hit)
best = max(choice, key=self._specific)
if best not in picked:
picked.append(best)
# The widest units named hold the rest: one country, else the regions
# of one country. Several countries are a list, not a path.
for kind in ("country", "region"):
named = [p for p in picked if self.places[p].kind == kind]
if len({self.country(p) for p in named}) == 1:
holds = self.under(named)
picked = [p for p in picked if p in holds or self.places[p].kind == "continent"
or any(p in self.ancestors(q) for q in named)]
if named:
break
wider = {a for p in picked for a in self.ancestors(p)}
picked = [p for p in picked if p not in wider]
if country is not None and not any(p in within or p in self.ancestors(country)
for p in picked):
picked = [country]
return picked
def lookup(self, token: str) -> frozenset[int] | None:
"""The places a filter token names, before what lies in them: `country:DK`
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
every place written so (a town, or a region or a country by name).
None when the gazetteer knows no such place."""
tok = parse(token)
if tok.kind == "country":
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
found = {self.by_key[key]} if key in self.by_key else set()
elif tok.kind == "region":
found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \
else set(self.aliases.get(tok.value, ()))
else:
found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())}
return frozenset(found) or None