Files
jobboard/app/places.py
T

1258 lines
57 KiB
Python

"""Places as a filter: what a board writes in `place`, matched by whole words.
A location line is free text: "København K", "Kgs. Lyngby", "Aarhus N, Region
Central Jutland, Denmark". A substring match finds `Lund` in `Billund` and
`Denmark` in nothing that only names a town. Here a place is folded (casefolded,
marks stripped, `ø` to `o`, `æ` to `ae`, `å` to `a`, punctuation to spaces) and
matched as whole words, so `Lund` is `lund` and no other word. A filter is one
of three tokens, and several tokens are joined by OR:
country:DK the role's country, or a line that names it while `country`
is empty or `EU` (a board that lists several offices)
region:skane any town or name of a region in `REGIONS`
place:malmo the town's fold and its aliases, so `copenhagen`,
`kobenhavn` and `kopenhamn` find each other
`matches` answers whether one role's folded place and country fit a token, in
Python: a region is over a hundred phrases, and as `LIKE` clauses that costs
several hundred milliseconds a query, as a set intersection on the words of a
line it costs a few. `register` gives a connection that answer as the SQL
function `place_match`, and `match_sql` returns the clause that calls it.
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
chip's text back from the address.
The filter and the gates no longer match words: a role's places are resolved
against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
and a token is the places under the place it names. `matches` stays the rule
for a `place:` token the gazetteer does not know. This module opens no
database; the caller hands over the places it holds, or its connection.
A fourth token finds remote roles by where their hire may live:
remote:country:DK a remote role whose area (`role_remote`, written by
`app.prep`) overlaps Denmark: Denmark or a place in
it, a place Denmark lies in (Europe, Øresund), or
Anywhere
remote:anywhere a remote role whose area is Anywhere, and no other
remote:any every remote role in the viewing agent's scope, its
`require` gates on country (`Store`, which knows the
agent, answers it)
Anywhere is a place of its own, outside the tree (`ANYWHERE`), so `under`
never reaches every role and the Place filter stays a filter. It has no
alias: only a remote area holds it, never a role's places.
"""
from __future__ import annotations
import re
import sqlite3
from collections import Counter
from collections.abc import Callable, Iterable, Sequence
from dataclasses import dataclass
from functools import lru_cache
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
country_of, remote_of)
__all__ = ["ANYWHERE", "ANYWHERE_WORDS", "KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place",
"Region", "Seed", "Suggestion", "Token", "fold", "known", "label", "match_sql",
"matches", "parse", "placeless", "register", "remote_part", "seed", "suggest",
"towns"]
def fold(text: str) -> str:
"""`Skåne län` -> `skane lan`, `København K` -> `kobenhavn k`, `Kgs. Lyngby`
-> `kgs lyngby`. What `roles.place_fold` holds. `å` needs no rule of its own:
NFKD splits it into `a` and a ring, and the ring is a mark."""
return _place_fold(text)
# ------------------------------------------------------------------ countries
def _countries() -> dict[str, tuple[str, ...]]:
"""Every code to the names `_COUNTRY_NAMES` gives it, English name first."""
found: dict[str, tuple[str, ...]] = {}
for line in _COUNTRY_NAMES.split("\n"):
if line:
code, rest = line.split(" ", 1)
found[code] = tuple(name.strip() for name in rest.split("|"))
return found
_COUNTRY_LABELS = _countries()
#: A code to the folded names a country goes by.
_COUNTRY_FOLDS: dict[str, tuple[str, ...]] = {
code: tuple(dict.fromkeys(filter(None, map(fold, names))))
for code, names in _COUNTRY_LABELS.items()}
#: What a line has to say for a role of that country whose `country` column is
#: empty or `EU`: a name of the country, or a city `country_of` knows in it. A
#: board that lists offices ("Copenhagen; London; Warsaw") names cities only.
_COUNTRY_MENTIONS: dict[str, tuple[str, ...]] = {
code: tuple(dict.fromkeys((*names, *(fold(c) for c, cc in _CITIES.items() if cc == code))))
for code, names in _COUNTRY_FOLDS.items()}
#: A folded country name to its code.
_COUNTRY_BY_NAME: dict[str, str] = {
name: code for code, names in _COUNTRY_FOLDS.items() for name in names}
#: Words that are not a town: a country, a macro region, how the role is worked.
_NOT_A_TOWN = frozenset(_COUNTRY_BY_NAME) | frozenset(map(fold, _REGIONS)) \
| {"remote", "hybrid"}
# -------------------------------------------------------------------- aliases
#: Towns boards write in more than one language, first spelling shown. Each
#: group is symmetric: any spelling finds the others. The Nordics and the DACH
#: countries in full for their large cities, and the European cities that
#: appear both ways in boards.
_ALIAS_GROUPS: tuple[tuple[str, ...], ...] = (
# Denmark
("Copenhagen", "København", "Kobenhavn", "Köpenhamn", "Kopenhamn", "Kopenhagen",
"Copenhague", "Kopenhaga"),
("Aarhus", "Århus"),
("Aalborg", "Ålborg"),
("Kgs. Lyngby", "Kongens Lyngby", "Lyngby"),
("Taastrup", "Tåstrup"),
("Elsinore", "Helsingør"),
# Sweden
("Gothenburg", "Göteborg", "Goteborg"),
("Stockholm",),
("Malmö", "Malmo"),
("Helsingborg", "Hälsingborg"),
("Västerås", "Vasteras"),
("Linköping", "Linkoping"),
("Örebro", "Orebro"),
("Norrköping", "Norrkoping"),
("Jönköping", "Jonkoping"),
# Norway, Finland, Iceland
("Oslo",),
("Bergen",),
("Trondheim", "Trondhjem"),
("Helsinki", "Helsingfors"),
("Espoo", "Esbo"),
("Vantaa", "Vanda"),
("Reykjavik", "Reykjavík"),
# Germany
("Munich", "München", "Muenchen", "Munchen"),
("Cologne", "Köln", "Koeln", "Koln"),
("Nuremberg", "Nürnberg", "Nuernberg", "Nurnberg"),
("Düsseldorf", "Duesseldorf", "Dusseldorf"),
("Hanover", "Hannover"),
("Mainz", "Mayence"),
("Frankfurt", "Frankfurt am Main"),
("Göttingen", "Goettingen", "Gottingen"),
("Würzburg", "Wuerzburg", "Wurzburg"),
("Tübingen", "Tuebingen", "Tubingen"),
("Lübeck", "Luebeck", "Lubeck"),
("Münster", "Muenster", "Munster"),
("Saarbrücken", "Saarbruecken", "Saarbrucken"),
("Neukirchen-Vluyn",),
# Austria
("Vienna", "Wien", "Vienne"),
("Salzburg",),
("Innsbruck",),
# Switzerland
("Zurich", "Zürich", "Zuerich"),
("Geneva", "Genève", "Geneve", "Genf", "Ginevra"),
("Basel", "Basle", "Bâle"),
("Bern", "Berne"),
("Lucerne", "Luzern"),
("St. Gallen", "Sankt Gallen", "Saint Gall"),
# The rest of Europe
("Prague", "Praha", "Prag"),
("Warsaw", "Warszawa", "Warschau"),
("Krakow", "Kraków", "Cracow"),
("Wroclaw", "Wrocław", "Breslau"),
("Gdansk", "Gdańsk", "Danzig"),
("Poznan", "Poznań", "Posen"),
("Lisbon", "Lisboa", "Lissabon"),
("Milan", "Milano", "Mailand"),
("Rome", "Roma", "Rom"),
("Turin", "Torino"),
("Florence", "Firenze"),
("Naples", "Napoli", "Neapel"),
("Venice", "Venezia", "Venedig"),
("Genoa", "Genova"),
("Brussels", "Bruxelles", "Brussel", "Brüssel"),
("Antwerp", "Antwerpen", "Anvers"),
("Ghent", "Gent", "Gand"),
("Bruges", "Brugge"),
("Liège", "Liege", "Luik", "Lüttich"),
("The Hague", "Den Haag", "s-Gravenhage"),
("Luxembourg", "Luxemburg"),
("Seville", "Sevilla"),
("Athens", "Athina", "Athen"),
("Bucharest", "București", "Bucuresti"),
("Belgrade", "Beograd"),
("Bratislava", "Pressburg"),
("Kyiv", "Kiev"),
("Cluj-Napoca", "Cluj"),
("Timișoara", "Timisoara"),
("Marseille", "Marseilles"),
("Bolzano", "Bozen"),
("Padua", "Padova"),
("Forlì",),
("L'Aquila",),
("Massa-Carrara", "Massa Carrara"),
# France, with the marks and hyphens a fold drops
("Aix-en-Provence",), ("Bourg-en-Bresse",), ("Boulogne-sur-Mer",), ("Chambéry",),
("Clermont-Ferrand",), ("La Roche-sur-Yon",), ("Les Sables-d'Olonne",), ("Loudéac",),
("Nîmes",), ("Orléans",), ("Rezé",), ("Saint-Brieuc",), ("Saint-Étienne",),
("Saint-Herblain",), ("Saint-Malo",), ("Saint-Nazaire",), ("Saint-Ouen-l'Aumône",),
# Germany, likewise
("Annaberg-Buchholz",), ("Baden-Baden",), ("Bietigheim-Bissingen",), ("Böblingen",),
("Donauwörth",), ("Flörsheim", "Flohrsheim"), ("Fürth",), ("Gießen", "Giessen"),
("Göppingen",), ("Großostheim",), ("Günzburg",), ("Güstrow",), ("Halle (Saale)",),
("Höxter",), ("Idar-Oberstein",), ("Königsbrunn",), ("Künzelsau",), ("Lörrach",),
("Lüchow",), ("Meißen", "Meissen"), ("Möhnesee",), ("Mönchengladbach",), ("Mühldorf",),
("Mülheim",), ("Neumünster",), ("Nördlingen",), ("Öhringen",), ("Ribnitz-Damgarten",),
("Röbel",), ("Schwäbisch Gmünd",), ("Schwäbisch Hall",), ("Überlingen",),
("Villingen-Schwenningen",), ("Waren an der Müritz",), ("Wolfenbüttel",),
# Elsewhere
("Cegléd",), ("Brașov", "Brasov"), ("Älmhult",), ("Hornbæk",), ("Melaka", "Malacca"),
("Samut Prakan", "Samut Prakarn"), ("Xi'an",), ("Ho Chi Minh City", "Ho Chi Minh"),
)
def _variants(text: str) -> tuple[str, ...]:
"""A fold and its `aa` spelling: Danish writes `å` as `aa` (`Aarhus`, `Taastrup`)
and `Århus` and `Tåstrup` fold to `arhus` and `tastrup`."""
return tuple(dict.fromkeys((text, text.replace("aa", "a"))))
def _aliases() -> tuple[dict[str, tuple[str, ...]], dict[str, str]]:
"""Every folded spelling to its whole group (all folded), and to its shown
spelling. The group starts with the first spelling listed."""
groups: dict[str, tuple[str, ...]] = {}
shown: dict[str, str] = {}
for group in _ALIAS_GROUPS:
members = tuple(dict.fromkeys(v for name in group for v in _variants(fold(name))))
for name in group:
shown.setdefault(fold(name), name)
for member in members:
groups[member] = members
return groups, shown
_GROUPS, _SHOWN = _aliases()
def _expand(text: str) -> tuple[str, ...]:
"""A folded town and everything it is also written as."""
return _GROUPS.get(text) or _variants(text)
# -------------------------------------------------------------------- regions
@dataclass(frozen=True)
class Region:
"""A region below a country, or across two. `names` are what a board calls
the region itself, `towns` the places in it; a line is in the region when it
has one of either as whole words, or is in a region it `includes`, by key.
Only `names` are what the region is suggested and read as. Spellings are
shown with their marks and folded when used."""
key: str
label: str
names: tuple[str, ...]
towns: tuple[str, ...]
country: str = ""
includes: tuple[str, ...] = ()
def _split(text: str) -> tuple[str, ...]:
return tuple(part.strip() for part in text.split(",") if part.strip())
_SKANE = Region(
"skane", "Skåne",
_split("Skåne, Skåne län, Skåne County, Region Skåne, Skåne Region, Scania, Skaane"),
_split("""Malmö, Lund, Helsingborg, Kristianstad, Landskrona, Trelleborg, Ängelholm,
Hässleholm, Ystad, Eslöv, Höganäs, Staffanstorp, Lomma, Bjuv, Burlöv, Kävlinge,
Svedala, Vellinge, Simrishamn, Tomelilla, Sjöbo, Svalöv, Klippan, Åstorp,
Bromölla, Osby, Perstorp, Örkelljunga, Östra Göinge, Hörby, Höör, Båstad,
Skurup, Arlöv, Limhamn, Dalby, Åkarp, Bjärred, Höllviken, Ljungbyhed,
Löddeköpinge, Furulund, Genarp, Veberöd, Skanör, Falsterbo, Mölle, Arild"""),
"SE")
_HOVEDSTADEN = Region(
"hovedstaden", "Capital Region of Denmark",
_split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"),
_split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup,
Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte,
Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre,
Høje-Taastrup, Hørsholm, Ishøj, Lyngby-Taarbæk, Rudersdal, Rødovre, Tårnby,
Vallensbæk, Kgs. Lyngby, Kongens Lyngby, Lyngby, Virum, Holte, Nærum, Birkerød,
Vedbæk, Skodsborg, Klampenborg, Charlottenlund, Hellerup, Ordrup, Bagsværd,
Søborg, Mørkhøj, Måløv, Skovlunde, Smørum, Ølstykke, Stenløse, Værløse, Farum,
Slangerup, Hedehusene, Taastrup, Kastrup, Amager, Valby, Vanløse, Brønshøj,
Nørrebro, Østerbro, Vesterbro, Nordhavn, Ørestad, Sydhavn, Islands Brygge,
Refshaleøen, Humlebæk, Kokkedal, Espergærde, Snekkersten, Rungsted, Rungsted Kyst,
Lillerød, Frederiksværk, Hundested, Helsinge, Gilleleje, Græsted, Vallensbæk Strand,
Skævinge, Rønne, Nexø, Allinge, Svaneke, Aakirkeby, Hasle"""),
"DK")
# Region Sjælland, not the island: Copenhagen is on Zealand too, but a board that
# writes "Region Zealand" means the region and the Capital Region has its own key.
# The bare word is left out of `names` since "New Zealand" would match it.
_ZEALAND = Region(
"zealand", "Zealand",
_split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"),
_split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved,
Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg,
Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing,
Haslev, Korsør, Skælskør, Præstø, Stege, Fakse, Rødby, Rødbyhavn, Jyllinge,
Viby Sjælland, Solrød Strand, Fuglebjerg, Gørlev, Dianalund, Jyderup, Hvalsø,
Tølløse, Hørve, Fårevejle, Bjæverskov, Nørre Asmindrup"""),
"DK")
_JUTLAND = Region(
"jutland", "Jutland",
_split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland,
North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland,
East Jutland, West Jutland, Vestjylland, Østjylland"""),
_split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning,
Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov,
Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå,
Hjørring, Frederikshavn, Skagen, Thisted, Struer, Ikast, Brande, Varde, Ribe,
Tønder, Vojens, Hadsten, Hobro, Skanderborg, Odder, Hedensted, Vejen, Ebeltoft,
Lemvig, Rønde, Egå, Lystrup, Hinnerup, Tilst, Hammel, Galten, Sabro, Harlev,
Løsning, Højbjerg, Juelsminde, Nørresundby, Brønderslev, Aars, Nibe, Sæby, Løgstør, Tarm,
Skjern, Padborg, Rødekro, Toftlund, Bramming, Kruså, Brabrand, Ry, Gjern"""),
"DK")
_FUNEN = Region(
"funen", "Funen",
_split("Funen, Fyn, Region Fyn"),
_split("""Odense, Svendborg, Nyborg, Faaborg, Fåborg, Middelfart, Assens, Kerteminde,
Bogense, Langeskov, Ringe, Otterup, Munkebo, Vissenbjerg, Årslev, Tommerup,
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
"DK")
# The Øresund region is the two capital regions either side of the sound. It is
# suggested on its own names only; a line in either region is in it.
_ORESUND = Region(
"oresund", "Øresund",
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"),
(), includes=("hovedstaden", "skane"))
# Germany's states and the areas boards write for a remote role's area, by
# name, towns not listed: a town in Germany is found through the country.
_GERMANY = tuple(Region(key, label, _split(names), (), "DE") for key, label, names in (
("niedersachsen", "Niedersachsen", "Niedersachsen, Lower Saxony"),
("norddeutschland", "Norddeutschland", "Norddeutschland, Northern Germany"),
("ostdeutschland", "Ostdeutschland", "Ostdeutschland, Eastern Germany"),
("suddeutschland", "Süddeutschland", "Süddeutschland, Southern Germany"),
("baden-wurttemberg", "Baden-Württemberg", "Baden-Württemberg"),
("bayern", "Bayern", "Bayern, Bavaria"),
("brandenburg", "Brandenburg", "Brandenburg"),
("hessen", "Hessen", "Hessen, Hesse"),
("mecklenburg-vorpommern", "Mecklenburg-Vorpommern",
"Mecklenburg-Vorpommern, Mecklenburg-Western Pomerania"),
("nordrhein-westfalen", "Nordrhein-Westfalen", "Nordrhein-Westfalen, North Rhine-Westphalia, NRW"),
("rheinland-pfalz", "Rheinland-Pfalz", "Rheinland-Pfalz, Rhineland-Palatinate"),
("saarland", "Saarland", "Saarland"),
("sachsen", "Sachsen", "Sachsen, Saxony"),
("sachsen-anhalt", "Sachsen-Anhalt", "Sachsen-Anhalt, Saxony-Anhalt"),
("schleswig-holstein", "Schleswig-Holstein", "Schleswig-Holstein"),
("thuringen", "Thüringen", "Thüringen, Thuringia"),
("bodensee", "Bodensee", "Bodensee, Lake Constance"),
("harz", "Harz", "Harz"),
("rugen", "Rügen", "Rügen"),
("uckermark", "Uckermark", "Uckermark")))
REGIONS: dict[str, Region] = {
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN, *_GERMANY)}
def _phrases(region: Region) -> tuple[str, ...]:
"""Every folded phrase that puts a line in the region, with the other
spellings of its towns, and those of the regions it includes."""
found: dict[str, None] = {}
for text in (*region.names, *region.towns):
for form in _expand(fold(text)):
found[form] = None
for key in region.includes:
found.update(dict.fromkeys(_phrases(REGIONS[key])))
return tuple(found)
_PHRASES: dict[str, tuple[str, ...]] = {key: _phrases(r) for key, r in REGIONS.items()}
#: A folded key, label or name to the region key; a name in two regions ("Capital
#: Region of Denmark" is also in Øresund) belongs to the one that lists it first.
_REGION_BY_NAME: dict[str, str] = {}
for _key, _region in REGIONS.items():
for _name in (_region.key, _region.label, *_region.names):
_REGION_BY_NAME.setdefault(fold(_name), _key)
#: What a place is shown as when a chip is read from the address: every alias
#: spelling, every region and every town of one, as written above.
_TOWN_LABELS: dict[str, str] = dict(_SHOWN)
for _region in REGIONS.values():
for _town in (_region.label, *_region.towns, *_region.names):
_TOWN_LABELS.setdefault(fold(_town), _town)
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
def _group(folded: str) -> str:
"""The key of the town a fold is: the first spelling of its alias group, else
the fold with `aa` as `a`."""
return _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
def _known_towns() -> dict[str, str]:
"""Every town the tables name, as a fold, to its group: the alias groups, the
towns of the regions and the cities `country_of` knows."""
names = [*_GROUPS, *(fold(city) for city in _CITIES),
*(fold(town) for region in REGIONS.values() for town in region.towns)]
return {form: _group(form) for name in names for form in _variants(name)}
_KNOWN = _known_towns()
_KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN)
# --------------------------------------------------------------------- tokens
@dataclass(frozen=True)
class Token:
"""One place filter. `value` is a country code, a region key or a fold;
for `remote`, the place token (`country:DK`), `anywhere` or `any`."""
kind: str
value: str
label: str
def __str__(self) -> str:
return f"{self.kind}:{self.value}"
def _region_key(text: str) -> str:
folded = fold(text)
return _REGION_BY_NAME.get(folded, folded)
#: The words that are Anywhere: in a remote token (`remote:anywhere`), and in
#: a place text that says remote ("Home Based - Worldwide").
ANYWHERE_WORDS = frozenset(("anywhere", "worldwide", "world wide"))
def _remote(value: str) -> Token:
"""`remote:<place token>`: `value` is the place token, whose label the
chip reads as "Remote · Denmark"; Europe (`country:EU`) is "Remote ·
Europe", `anywhere` "Remote · anywhere" and `any` "Remote". A remote
token inside one names nothing."""
if fold(value) == "any":
return Token("remote", "any", "Remote")
if fold(value) in ANYWHERE_WORDS:
return Token("remote", "anywhere", "Remote · anywhere")
inner = parse(value)
if inner.kind == "remote" or not value:
return Token("remote", "", "Remote")
shown = "Europe" if (inner.kind, inner.value) == ("country", "EU") else inner.label
return Token("remote", str(inner), f"Remote · {shown}")
def parse(token: str) -> Token:
"""`country:DK`, `region:skane`, `place:malmo` or `remote:<one of these>`.
Bare text, or a prefix that is none of these, is a place: `Skåne` is
`place:skane`. A country by name (`country:Denmark`) and a region by label
(`region:Skåne`) are read as well."""
kind, sep, value = (token or "").partition(":")
kind = kind.strip().lower()
if sep and kind == "remote":
return _remote(value.strip())
if not sep or kind not in ("country", "region", "place"):
kind, value = "place", token or ""
value = value.strip()
if kind == "country":
code = value.upper()
if code not in _COUNTRY_LABELS:
code = _COUNTRY_BY_NAME.get(fold(value), code)
return Token("country", code, _COUNTRY_LABELS.get(code, (code,))[0])
if kind == "region":
key = _region_key(value)
return Token("region", key, REGIONS[key].label if key in REGIONS else key)
folded = fold(value)
return Token("place", folded, _TOWN_LABELS.get(folded) or folded.title())
def label(token: str | Token) -> str:
"""The text of a chip read from the address: `country:DK` is `Denmark`,
`region:skane` is `Skåne`, `place:malmo` is `Malmö`, `remote:country:DK`
is `Remote · Denmark`. A town the tables do not know is its fold in title
case."""
return (token if isinstance(token, Token) else parse(token)).label
# ------------------------------------------------------------------- matching
SQL_FUNCTION = "place_match"
def known(token: str) -> bool:
"""Whether the token names something: a country of the table (or `EU`), a
region of `REGIONS`, a place with a value, or `remote:` and one of these,
`anywhere` or `any`. `region:atlantis`, `country:ZZ`, `place:` and
`remote:country:ZZ` are not."""
return _known(parse(token))
def _known(tok: Token) -> bool:
if tok.kind == "remote":
return tok.value in ("any", "anywhere") or (bool(tok.value) and _known(parse(tok.value)))
if tok.kind == "country":
return tok.value in _COUNTRY_LABELS or tok.value == "EU"
if tok.kind == "region":
return tok.value in REGIONS
return bool(tok.value)
@dataclass(frozen=True)
class _Matcher:
"""What a token asks of a role. `phrases` are whole-word phrases of the
folded place; `countries` are `(code, mentions)`: the role's own country, or
a line with one of the mentions while `country` is empty or `EU`, since a
board listing several offices has no single country."""
phrases: frozenset[str]
countries: tuple[tuple[str, frozenset[str]], ...]
longest: int
@lru_cache(maxsize=1024)
def _matcher(token: str) -> _Matcher | None:
"""The token read once: `None` when it names nothing.
A `place:` value that is also a region or a country is matched as that too:
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
written as a bare word means what the owner meant by it."""
tok = parse(token)
if not _known(tok) or tok.kind == "remote":
return None
phrases: set[str] = set()
countries: list[tuple[str, frozenset[str]]] = []
if tok.kind == "country":
countries.append((tok.value, frozenset(_COUNTRY_MENTIONS.get(tok.value, ()))))
elif tok.kind == "region":
phrases.update(_PHRASES[tok.value])
else:
phrases.update(_expand(tok.value))
phrases.update(_PHRASES.get(_REGION_BY_NAME.get(tok.value, ""), ()))
if code := _COUNTRY_BY_NAME.get(tok.value):
countries.append((code, frozenset(_COUNTRY_MENTIONS.get(code, ()))))
phrases.discard("")
longest = max((len(p.split()) for p in (*phrases, *(m for _, ms in countries for m in ms))),
default=1)
return _Matcher(frozenset(phrases), tuple(countries), longest)
@lru_cache(maxsize=32768)
def _runs(place_fold: str, longest: int) -> frozenset[str]:
"""Every run of up to `longest` words of a folded line, each a whole-word
phrase the line holds."""
words = place_fold.split()
return frozenset(" ".join(words[i:i + n])
for n in range(1, longest + 1) for i in range(len(words) - n + 1))
def matches(place_fold: str, country: str, token: str) -> bool:
"""Whether a role whose folded place and country are these is found by the
token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or
a Danish city) while `country` is empty or `EU`. `region:skane` is any of
its names or towns as whole words, `place:malmo` the town and its aliases.
A token that names nothing (`known` is false) matches no role, nor does a
`remote:` token, which only the gazetteer's areas answer."""
match = _matcher(token)
if match is None:
return False
runs = None
if match.phrases:
runs = _runs(place_fold, match.longest)
if not match.phrases.isdisjoint(runs):
return True
for code, mentions in match.countries:
if country == code:
return True
if mentions and country in ("", "EU"):
runs = _runs(place_fold, match.longest) if runs is None else runs
if not mentions.isdisjoint(runs):
return True
return False
def _place_match(place_fold: str | None, country: str | None, token: str) -> int:
return int(matches(place_fold or "", country or "", token))
def register(conn: sqlite3.Connection) -> None:
"""Give a connection `place_match(place_fold, country, token)`, which is 1 when
`matches` is true. `match_sql` returns a clause that calls it."""
conn.create_function(SQL_FUNCTION, 3, _place_match, deterministic=True)
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
-> tuple[str, list]:
"""SQL over the two columns for one token, with one `?` parameter. It calls
`place_match`, so the connection needs `register`. Several tokens are joined
with OR by the caller. Bare text is a place. A token that names nothing
(`known` is false) is `0`, which matches no role."""
tok = parse(token)
if not _known(tok) or tok.kind == "remote":
return "0", []
return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)]
# ---------------------------------------------------------------- suggestions
@dataclass(frozen=True)
class Suggestion:
"""`token` is what the chip holds, `kind` is `country`, `region`, `place`
or `remote`, `count` the open roles it stands for."""
token: str
label: str
kind: str
count: int
def _prefixes(q: str, candidates: Iterable[str]) -> bool:
"""`q` starts a word of a candidate, or a run of words: `kgs ly` finds
`kgs lyngby`, `hague` finds `the hague`."""
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
#: Where one town ends in a line: `;` and `|` between places, and `or` / `eller` /
#: `og` between two towns in one ("København V eller Aalborg").
_PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Danish boards append "and the possibility of working from home".
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
#: A slash between two places, "Malmö/Lund", "Berlin / Munich".
_SLASH = re.compile(r"\s*/\s*")
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
#: A country or "Remote" ahead of the town, "Sweden - Lund", "Remote-New Zealand".
_LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$")
def remote_part(text: str) -> bool:
"""Whether a place text, or one part of it, says the role is worked
remotely: "Germany (Remote)", "Remote - US", "Home Based - EMEA",
"Worldwide". One that says hybrid too is not ("Remote or hybrid,
Berlin"), as `remote_of` reads hybrid first."""
return remote_of(text) == "remote"
def _towns(place: str) -> list[str]:
"""The first comma part of each place of a line: `Aarhus N, Region Central
Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal
district is dropped, since `place:kobenhavn` finds `København K` anyway.
A part that says remote (`remote_part`) names where a hire may live, not
a town: `USA (Remote)` and `Remote Germany` give none."""
found = []
for part in _PART.split(_AND_HOME.sub("", place or "")):
if remote_part(part):
continue
head = part.split(",")[0].strip()
if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN:
head = lead[2]
head = _DISTRICT.sub("", head)
if fold(head) and fold(head) not in _NOT_A_TOWN:
found.append(head)
return found
def _towns_in(folded: str) -> list[str]:
"""The known towns a fold holds as whole words, longest first from the left,
as their groups: `munich berlin` is Munich and Berlin, `copenhagen office` is
Copenhagen, `hoje taastrup` is one town and not also Taastrup."""
words = folded.split()
found: list[str] = []
i = 0
while i < len(words):
for n in range(min(_KNOWN_LONGEST, len(words) - i), 0, -1):
if (group := _KNOWN.get(" ".join(words[i:i + n]))) is not None:
found.append(group)
i += n
break
else:
i += 1
return list(dict.fromkeys(found))
def towns(place: str) -> frozenset[str]:
"""The towns a line names, each as its group's key, so `København K` and
`Copenhagen, Denmark` are both `{"copenhagen"}`: the first comma part of each
place, and every known town the line holds as whole words, so a list such as
`Aalborg, Copenhagen, Kolding` gives all three. A state or province after
the town is not a town, so `Mountain View, California` and `San Francisco,
California` share none. Empty for a line that names only a country, a region
or how the role is worked."""
found = {_group(fold(town)) for town in _towns(place)}
found.update(_towns_in(fold(place or "")))
return frozenset(found)
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
"""Open roles per region, over folded lines. A line is cut into every run of
words once, and each region asks whether it holds one of its phrases."""
sets = {key: frozenset(_PHRASES[key]) for key in keys}
longest = max((len(p.split()) for phrases in sets.values() for p in phrases), default=1)
counts: Counter[str] = Counter()
for text, roles in places:
runs = _runs(text, longest)
for key, phrases in sets.items():
if not phrases.isdisjoint(runs):
counts[key] += roles
return counts
#: The kinds a remote suggestion is offered for by name alone; a region or a
#: town only when a remote area names it or a place in it.
_WIDE = ("continent", "country")
def _asks_remote(text: str) -> tuple[bool, str]:
"""Whether typed text asks for the remote tokens as such, and the text of
a place after "remote": `remo` is `(True, "")`, `remote den` `(True,
"den")`, `den` `(False, "den")`. Three letters at least, so "re" is
still Regensburg."""
head, _, rest = text.partition(" ")
if head == "remote":
return True, rest.strip()
return (True, "") if len(text) >= 3 and "remote".startswith(text) else (False, text)
def _remote_suggestions(text: str, gaz: Gazetteer,
areas: list[tuple[tuple[int, ...], int]],
scope: Iterable[int] = ()) -> list[Suggestion]:
"""`remote:` tokens for typed text, each with the open remote roles it
finds (`Gazetteer.overlap`, which holds Anywhere), none that finds
nothing, widest first and then by open roles. A place is offered when it
is a continent or a country, or a remote area names it or a place in it
(`held`). "remote" alone offers every continent and country held;
"remote den" or "den" the places offered whose name it starts, a region
or a town only when its token names it alone. `remote:anywhere` is a
token and no suggestion: every remote chip holds Anywhere already. With
a `scope`, the places of the asking agent's `require` gates on country,
only a place in the scope or one the scope lies in is offered, so an
agent kept to Europe is not offered Remote · United States."""
asked, text = _asks_remote(text)
held: set[int] = set()
for ids, _ in areas:
for place_id in ids:
held |= {place_id, *gaz.ancestors(place_id)}
if asked and not text:
candidates = {p for p in held if gaz.places[p].kind in _WIDE}
elif text:
candidates = {p for p, place in gaz.places.items()
if (place.kind in _WIDE or p in held) and _prefixes(text, gaz.names(p))}
else:
candidates = set()
if scope := set(scope):
candidates &= gaz.under(scope) | {a for p in scope for a in gaz.ancestors(p)}
found: dict[str, tuple[int, Suggestion]] = {}
for place_id in candidates - {gaz.anywhere}:
token = parse(f"remote:{gaz.token(place_id)}")
named = gaz.lookup(str(token))
if str(token) in found or not named or place_id not in named \
or (token.value.startswith("place:") and len(named) > 1):
continue
wanted = gaz.overlap(named)
if count := sum(n for ids, n in areas if not wanted.isdisjoint(ids)):
found[str(token)] = (KINDS.index(gaz.places[place_id].kind),
Suggestion(str(token), token.label, "remote", count))
return [s for _, s in sorted(found.values(), key=lambda f: (f[0], -f[1].count, f[1].label))]
def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12,
remote: Iterable[tuple[Iterable[int], int]] = (),
gaz: Gazetteer | None = None,
remote_any: Callable[[], int] | None = None,
scope: Iterable[int] = ()) -> list[Suggestion]:
"""Tokens for the text typed so far. `places` is `(place, country, open
roles)`, `remote` is `(area, open roles)` for the open remote roles, the
area as place ids of `gaz`, and `remote_any` counts the open roles
`remote:any` finds for the agent asking, called only when it is offered.
`remote:any` ("Remote") first when `q` starts "remote", then countries,
then the other remote tokens, then regions, then towns, each by open
roles.
A country is suggested when `q` starts a word of one of its names, with the
roles that carry its code. A region likewise on its label and names, with the
roles whose line is in it. A town is the first comma part of each `;` part of
a line, grouped by fold and by alias (`Copenhagen` and `København` are one
town, shown as its commonest spelling). Countries, regions and towns with no
open role are left out: a chip that finds nothing helps nobody. Remote
tokens are `_remote_suggestions`, offered only with `gaz` and kept to
`scope` when one is given."""
text = fold(q)
if not text:
return []
rows = [(place or "", country or "", int(roles)) for place, country, roles in places]
by_country: Counter[str] = Counter()
for _, country, roles in rows:
by_country[country] += roles
countries = [Suggestion(f"country:{code}", names[0], "country", by_country[code])
for code, names in _COUNTRY_LABELS.items()
if by_country[code] and _prefixes(text, _COUNTRY_FOLDS[code])]
named = [key for key, r in REGIONS.items()
if _prefixes(text, [fold(r.label), *map(fold, r.names)])]
counts = _region_counts(named, [(fold(place), roles) for place, _, roles in rows])
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
for key in named if counts[key]]
# A line's town is a group. A line that is a town is that group's spelling;
# one that only holds a known town ("Copenhagen Office", "Munich - Berlin")
# counts under each such town and shows as none; anything else is its own.
counts_by_town: Counter[str] = Counter()
spellings: dict[str, Counter[str]] = {}
written: dict[str, set[str]] = {}
for place, _, roles in rows:
for town in _towns(place):
folded = fold(town)
if folded not in _KNOWN and (inside := _towns_in(folded)):
for group in inside:
counts_by_town[group] += roles
continue
group = _group(folded)
counts_by_town[group] += roles
spellings.setdefault(group, Counter())[town] += roles
written.setdefault(group, set()).add(folded)
towns = []
for group, total in counts_by_town.items():
forms = {group, *written.get(group, ())}
if not _prefixes(text, [form for folded in forms for form in _expand(folded)]):
continue
seen = spellings.get(group)
shown = min(seen, key=lambda s: (-seen[s], s)) if seen \
else _TOWN_LABELS.get(group) or group.title()
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total))
remotes = _remote_suggestions(text, gaz, [(tuple(ids), int(n)) for ids, n in remote],
scope) if gaz is not None else []
first = []
if remote_any is not None and _asks_remote(text)[0] and (count := remote_any()):
first.append(Suggestion("remote:any", "Remote", "remote", count))
countries.sort(key=lambda s: (-s.count, s.label))
regions.sort(key=lambda s: (-s.count, s.label))
towns.sort(key=lambda s: (-s.count, s.label))
return (first + countries + remotes + regions + towns)[:limit]
# ------------------------------------------------------------------ gazetteer
#: The kinds of place, widest first: the index is how specific a kind is.
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
#: `(kind, key)` of Anywhere, where a remote hire may live when a posting says
#: "worldwide". A continent with no parent and nothing in it, so `under` never
#: reaches every role.
ANYWHERE = ("continent", "world")
#: What is Anywhere in a part of a place text that says remote: "Remote,
#: Global" is Anywhere, while "Global HQ, Berlin" is Berlin.
_ANYWHERE_IF_REMOTE = frozenset(("global", "international", *ANYWHERE_WORDS))
#: Entries of `_CITIES` that name a region rather than a town.
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
"bay area", "emilia romagna", "tuscany", "lazio"))
#: Words a place text holds when it names how or where-ever a role is worked
#: rather than a place: "Remote", "Multiple Locations", "Home based". A text
#: of these alone is no place, so it is never listed as unresolved.
_PLACELESS = frozenset((
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
def placeless(text: str) -> bool:
"""Whether a place text names no place at all: empty, or only numbers and
words such as `Remote`, `Hybrid`, `2 Locations`."""
return all(word in _PLACELESS or word.isdigit() for word in fold(text).split())
@dataclass(frozen=True)
class Seed:
"""One place as the code knows it: `key` is unique within `kind` (an ISO
code for a country, a region's key, a town's group), `parent` the
`(kind, key)` of the place it lies in, `aliases` the folds it is written
as."""
kind: str
key: str
name: str
parent: tuple[str, str] | None
aliases: tuple[str, ...]
def _subdivisions() -> list[tuple[str, str, str]]:
"""`(country, code, name)` of every US state and Canadian province."""
found = []
for line in _SUBDIVISIONS.split("\n"):
if line:
country, rest = line.split(" ", 1)
for entry in rest.split("|"):
name, code = entry.strip().rsplit(" ", 1)
found.append((country, code, name))
return found
def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
"""The gazetteer the dictionaries in code describe, parents before their
children, and the inclusions beyond the tree (Øresund holds the Capital
Region and Skåne, which lie in two countries).
Europe is a continent over the countries of `_EUROPE`, and is also what
`EU`, `EMEA` and `European Union` name. Asia Pacific (`APAC`) is a continent over
nothing. Every country of `_COUNTRY_NAMES`
by its names; the US states and Canadian provinces as regions; the
regions of `REGIONS`; the towns of the alias groups, of the regions and
of `_CITIES`, one place per group (`Copenhagen`, `København`,
`Kopenhamn` are one), in their region when a region lists them and else
in their country. Anywhere (`ANYWHERE`) stands apart, with no alias."""
europe = ("continent", "europe")
seeds: dict[tuple[str, str], Seed] = {}
def add(kind: str, key: str, name: str, parent: tuple[str, str] | None,
aliases: Iterable[str]) -> None:
folds = tuple(dict.fromkeys(a for a in aliases if a))
if (kind, key) in seeds:
old = seeds[(kind, key)]
folds = tuple(dict.fromkeys((*old.aliases, *folds)))
seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds)
else:
seeds[(kind, key)] = Seed(kind, key, name, parent, folds)
add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS)))
# Asia Pacific holds no country: a role placed there alone keeps the
# host's country, and one beside a country is that country.
add("continent", "asia-pacific", "Asia Pacific", None, ("asia pacific", "apac"))
add(*ANYWHERE, "Anywhere", None, ())
for code, names in _COUNTRY_LABELS.items():
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
for country, code, name in _subdivisions():
add("region", f"{country}-{code}", name, ("country", country), (fold(name),))
for region in REGIONS.values():
parent = ("country", region.country) if region.country else europe
add("region", region.key, region.label, parent,
(v for name in (region.key, region.label, *region.names)
for v in _variants(fold(name))))
for name in sorted(_CITY_REGIONS):
add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,))
# Towns, one per group, in the region that lists them first.
town_region: dict[str, str] = {}
for region in REGIONS.values():
for name in region.towns:
town_region.setdefault(_group(fold(name)), region.key)
folds: dict[str, dict[str, None]] = {}
shown: dict[str, str] = {}
def add_town(folded: str, name: str) -> None:
key = _group(folded)
folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded)))
shown.setdefault(key, name)
for spellings in _ALIAS_GROUPS:
for spelling in spellings:
add_town(fold(spelling), spellings[0])
for region in REGIONS.values():
for name in region.towns:
add_town(fold(name), name)
for city in _CITIES:
if city not in _CITY_REGIONS and city not in _REGION_BY_NAME:
add_town(city, _TOWN_LABELS.get(city) or city.title())
for key, forms in folds.items():
country = next((_CITIES[f] for f in forms if f in _CITIES), "")
if key in town_region:
parent = ("region", town_region[key])
elif country:
parent = ("country", country)
else:
raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES")
add("city", key, shown[key], parent, forms)
includes = [(("region", region.key), ("region", part))
for region in REGIONS.values() for part in region.includes]
return list(seeds.values()), includes
@dataclass(frozen=True)
class Place:
"""A row of `places`."""
id: int
kind: str
key: str
name: str
parent_id: int | None
class Gazetteer:
"""The places, their aliases and what lies in what, as the database holds
them, read into memory: what resolves a role's place text to places and a
filter token to the places under it. Built by the store from its tables,
and rebuilt when they change."""
def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]],
aliases: Iterable[tuple[str, int]],
includes: Iterable[tuple[int, int]] = ()) -> None:
self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows}
self.by_key: dict[tuple[str, str], int] = {
(p.kind, p.key): p.id for p in self.places.values()}
by_fold: dict[str, list[int]] = {}
for folded, place_id in aliases:
by_fold.setdefault(folded, []).append(place_id)
self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()}
self._longest = max((len(f.split()) for f in self.aliases), default=1)
self._up: dict[int, set[int]] = {}
self._down: dict[int, set[int]] = {}
for p in self.places.values():
if p.parent_id is not None:
self._link(p.parent_id, p.id)
for whole, part in includes:
self._link(whole, part)
self._ancestors: dict[int, frozenset[int]] = {}
self._resolved: dict[str, tuple[int, ...]] = {}
self._overlap: dict[frozenset[int], frozenset[int]] = {}
self._names: dict[int, tuple[str, ...]] | None = None
self.anywhere: int | None = self.by_key.get(ANYWHERE)
def _link(self, whole: int, part: int) -> None:
self._up.setdefault(part, set()).add(whole)
self._down.setdefault(whole, set()).add(part)
def ancestors(self, place_id: int) -> frozenset[int]:
"""Every place `place_id` lies in, through parents and inclusions."""
if (found := self._ancestors.get(place_id)) is None:
seen: set[int] = set()
todo = list(self._up.get(place_id, ()))
while todo:
up = todo.pop()
if up not in seen:
seen.add(up)
todo += self._up.get(up, ())
found = self._ancestors[place_id] = frozenset(seen)
return found
def under(self, ids: Iterable[int]) -> frozenset[int]:
"""The places and every place that lies in one of them."""
seen: set[int] = set()
todo = list(ids)
while todo:
place_id = todo.pop()
if place_id not in seen:
seen.add(place_id)
todo += self._down.get(place_id, ())
return frozenset(seen)
def overlap(self, ids: Iterable[int]) -> frozenset[int]:
"""Every place a remote area may hold to overlap these: the places,
what lies under them, every place one of those lies in, and Anywhere.
Denmark's holds Europe, and Øresund through the Capital Region, so a
role remote within Europe is open to someone in Denmark."""
key = frozenset(ids)
if (found := self._overlap.get(key)) is None:
under = self.under(key)
wider = set(under)
for place_id in under:
wider |= self.ancestors(place_id)
if self.anywhere is not None:
wider.add(self.anywhere)
found = self._overlap[key] = frozenset(wider)
return found
def names(self, place_id: int) -> tuple[str, ...]:
"""The folds a place is written as: its aliases and its name."""
if self._names is None:
by_id: dict[int, dict[str, None]] = {}
for folded, found in self.aliases.items():
for one in found:
by_id.setdefault(one, {})[folded] = None
for place in self.places.values():
by_id.setdefault(place.id, {})[fold(place.name)] = None
self._names = {place_id: tuple(folds) for place_id, folds in by_id.items()}
return self._names.get(place_id, ())
def token(self, place_id: int) -> str:
"""The filter token that names a place: `country:DK`, `country:EU`
for Europe, `region:skane` for a region of `REGIONS`, `anywhere` for
Anywhere, else `place:` and the name folded."""
place = self.places[place_id]
if place_id == self.anywhere:
return "anywhere"
if (place.kind, place.key) == ("continent", "europe"):
return "country:EU"
if place.kind == "country":
return f"country:{place.key}"
if place.kind == "region" and place.key in REGIONS:
return f"region:{place.key}"
return f"place:{fold(place.name)}"
def remote_places(self, text: str, marked: bool = False) -> tuple[int, ...]:
"""The places the parts of a place text that say remote
(`remote_part`) name, where a remote hire may live: `Germany
(Remote)` is Germany, `Cardiff, London or Remote (UK)` the United
Kingdom, `Home Based - Worldwide` and `Remote, Global` Anywhere.
`marked` takes every part, for a name the model read as where a
remote hire may live ("Europe", "anywhere").
`global` and `international` are Anywhere only here, so `Global HQ,
Berlin` stays Berlin; beside another place Anywhere is dropped, so
`Anywhere in France, Belgium` is the two countries. `Remote` alone
names none."""
found: list[int] = []
for part in _PART.split(_AND_HOME.sub("", text or "")):
if not (marked or remote_part(part)):
continue
ids = [p for p in self.resolve(part) if p != self.anywhere]
if self.anywhere is not None and any(
f" {word} " in f" {fold(part)} " for word in _ANYWHERE_IF_REMOTE):
ids.append(self.anywhere)
if len(ids) > 1 and self.anywhere in ids:
ids.remove(self.anywhere)
found += ids
return tuple(dict.fromkeys(found))
def area_label(self, ids: Sequence[int]) -> str:
"""A remote area as a row shows it: the names in order, Anywhere as
"anywhere", "United States, Canada". More than three places are
shown as the countries they lie in, so twenty German states read
"Germany"."""
found = [p for p in ids if p in self.places]
if len(found) > 3:
found = list(dict.fromkeys(
self.by_key.get(("country", self.country(p)), p) for p in found))
return ", ".join("anywhere" if p == self.anywhere else self.places[p].name
for p in found)
def country(self, place_id: int) -> str:
"""The ISO code of the country a place is in, '' above or outside one."""
place: Place | None = self.places.get(place_id)
while place is not None and place.kind != "country":
place = self.places.get(place.parent_id) if place.parent_id is not None else None
return place.key if place is not None else ""
def codes(self, ids: Iterable[int]) -> frozenset[str]:
"""The ISO codes of the countries among `ids`, and `EU` with Europe:
what a `roles.country` column holds for a role of one of them."""
found = set()
for place_id in ids:
place = self.places[place_id]
if place.kind == "country":
found.add(place.key)
elif (place.kind, place.key) == ("continent", "europe"):
found.add("EU")
return frozenset(found)
def _specific(self, place_id: int) -> int:
return KINDS.index(self.places[place_id].kind)
def _hits(self, folded: str) -> list[tuple[int, ...]]:
words = folded.split()
hits = []
i = 0
while i < len(words):
for n in range(min(self._longest, len(words) - i), 0, -1):
if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None:
hits.append(ids)
i += n
break
else:
i += 1
return hits
def resolve(self, text: str) -> tuple[int, ...]:
"""The places one place text names, most specific first from the left:
`Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund`
is Malmö and Lund, `Ellwangen, Germany` is Germany, `Remote` is none.
Every alias the text holds as whole words is a candidate, longest
first. A fold that names several places is the one in the country
`country_of` reads off the text, if one is, and else the most specific
(`Luxembourg` the town before the country). A candidate that another
one lies in is dropped, and so is one outside the one country the text
names, or, naming none, outside the regions it names in one country:
`London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the
Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several
countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country
`country_of` reads (from a state code, `Paris, TX`, or a code in
parentheses, `Remote (DE)`), the text is that country."""
folded = fold(text)
if (found := self._resolved.get(folded)) is None:
parts = _SLASH.split(_AND_HOME.sub("", text or ""))
found = tuple(dict.fromkeys(
p for part in parts for sub in _PART.split(part) for p in self._resolve(sub)))
self._resolved[folded] = found
return found
def _resolve(self, text: str) -> list[int]:
"""`resolve` for one part of a text, cut where `or`, `/` or `;` end one
place."""
code = country_of(text)
country = self.by_key.get(("country", code)) if code and code != "EU" else None
within = self.under((country,)) if country is not None else frozenset()
picked: list[int] = []
for hit in self._hits(fold(text)):
choice = [p for p in hit if p in within] or list(hit)
best = max(choice, key=self._specific)
if best not in picked:
picked.append(best)
# The widest units named hold the rest: one country, else the regions
# of one country. Several countries are a list, not a path.
for kind in ("country", "region"):
named = [p for p in picked if self.places[p].kind == kind]
if len({self.country(p) for p in named}) == 1:
holds = self.under(named)
picked = [p for p in picked if p in holds or self.places[p].kind == "continent"
or any(p in self.ancestors(q) for q in named)]
if named:
break
wider = {a for p in picked for a in self.ancestors(p)}
picked = [p for p in picked if p not in wider]
if country is not None and not any(p in within or p in self.ancestors(country)
for p in picked):
picked = [country]
return picked
def lookup(self, token: str) -> frozenset[int] | None:
"""The places a filter token names, before what lies in them: `country:DK`
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
every place written so (a town, or a region or a country by name).
A `remote:` token names the places of its place token, `remote:anywhere`
Anywhere, `remote:any` none. None when the gazetteer knows no such
place."""
tok = parse(token)
if tok.kind == "remote":
if tok.value != "anywhere":
return self.lookup(tok.value) if tok.value not in ("", "any") else None
found = {self.anywhere} if self.anywhere is not None else set()
elif tok.kind == "country":
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
found = {self.by_key[key]} if key in self.by_key else set()
elif tok.kind == "region":
found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \
else set(self.aliases.get(tok.value, ()))
else:
found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())}
return frozenset(found) or None