Match places with a registered SQL function, suggest regions on their own names, fold lines that hold a town into it
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
ebb1f560de
commit
1898f2a8a6
2 files changed
+947
-86
No files matched your search
+181
-64
@@ -13,23 +13,29 @@ of three tokens, and several tokens are joined by OR:
|
||||
place:malmo the town's fold and its aliases, so `copenhagen`,
|
||||
`kobenhavn` and `kopenhamn` find each other
|
||||
|
||||
`match_sql` turns a token into SQL over `roles.place_fold` and `roles.country`;
|
||||
`matches` answers whether one role's folded place and country fit a token, in
|
||||
Python: a region is over a hundred phrases, and as `LIKE` clauses that costs
|
||||
several hundred milliseconds a query, as a set intersection on the words of a
|
||||
line it costs a few. `register` gives a connection that answer as the SQL
|
||||
function `place_match`, and `match_sql` returns the clause that calls it.
|
||||
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
|
||||
chip's text back from the address. No database access here, the caller hands
|
||||
over the places it holds.
|
||||
chip's text back from the address. This module opens no database; the caller
|
||||
hands over the places it holds, or its connection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
|
||||
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
|
||||
|
||||
__all__ = ["REGIONS", "Region", "Suggestion", "Token", "fold", "label", "match_sql",
|
||||
"parse", "suggest"]
|
||||
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
|
||||
"label", "match_sql", "matches", "parse", "register", "suggest"]
|
||||
|
||||
|
||||
def fold(text: str) -> str:
|
||||
@@ -202,14 +208,16 @@ def _expand(text: str) -> tuple[str, ...]:
|
||||
class Region:
|
||||
"""A region below a country, or across two. `names` are what a board calls
|
||||
the region itself, `towns` the places in it; a line is in the region when it
|
||||
has one of either as whole words. Spellings are shown with their marks and
|
||||
folded when used."""
|
||||
has one of either as whole words, or is in a region it `includes`, by key.
|
||||
Only `names` are what the region is suggested and read as. Spellings are
|
||||
shown with their marks and folded when used."""
|
||||
|
||||
key: str
|
||||
label: str
|
||||
names: tuple[str, ...]
|
||||
towns: tuple[str, ...]
|
||||
country: str = ""
|
||||
includes: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _split(text: str) -> tuple[str, ...]:
|
||||
@@ -281,12 +289,12 @@ _FUNEN = Region(
|
||||
Ærø, Ærøskøbing, Langeland, Rudkøbing, Tåsinge, Marstal, Nordfyn"""),
|
||||
"DK")
|
||||
|
||||
# The Øresund region is the two capital regions either side of the sound.
|
||||
# The Øresund region is the two capital regions either side of the sound. It is
|
||||
# suggested on its own names only; a line in either region is in it.
|
||||
_ORESUND = Region(
|
||||
"oresund", "Øresund",
|
||||
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen")
|
||||
+ _HOVEDSTADEN.names + _SKANE.names,
|
||||
_HOVEDSTADEN.towns + _SKANE.towns)
|
||||
_split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"),
|
||||
(), includes=("hovedstaden", "skane"))
|
||||
|
||||
REGIONS: dict[str, Region] = {
|
||||
r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN)}
|
||||
@@ -294,11 +302,13 @@ REGIONS: dict[str, Region] = {
|
||||
|
||||
def _phrases(region: Region) -> tuple[str, ...]:
|
||||
"""Every folded phrase that puts a line in the region, with the other
|
||||
spellings of its towns."""
|
||||
spellings of its towns, and those of the regions it includes."""
|
||||
found: dict[str, None] = {}
|
||||
for text in (*region.names, *region.towns):
|
||||
for form in _expand(fold(text)):
|
||||
found[form] = None
|
||||
for key in region.includes:
|
||||
found.update(dict.fromkeys(_phrases(REGIONS[key])))
|
||||
return tuple(found)
|
||||
|
||||
|
||||
@@ -321,6 +331,24 @@ for _region in REGIONS.values():
|
||||
_NOT_A_TOWN |= frozenset(_REGION_BY_NAME)
|
||||
|
||||
|
||||
def _group(folded: str) -> str:
|
||||
"""The key of the town a fold is: the first spelling of its alias group, else
|
||||
the fold with `aa` as `a`."""
|
||||
return _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
|
||||
|
||||
|
||||
def _known_towns() -> dict[str, str]:
|
||||
"""Every town the tables name, as a fold, to its group: the alias groups, the
|
||||
towns of the regions and the cities `country_of` knows."""
|
||||
names = [*_GROUPS, *(fold(city) for city in _CITIES),
|
||||
*(fold(town) for region in REGIONS.values() for town in region.towns)]
|
||||
return {form: _group(form) for name in names for form in _variants(name)}
|
||||
|
||||
|
||||
_KNOWN = _known_towns()
|
||||
_KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- tokens
|
||||
|
||||
|
||||
@@ -369,57 +397,118 @@ def label(token: str | Token) -> str:
|
||||
return (token if isinstance(token, Token) else parse(token)).label
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ SQL
|
||||
# ------------------------------------------------------------------- matching
|
||||
|
||||
_NEVER = ("0", [])
|
||||
SQL_FUNCTION = "place_match"
|
||||
|
||||
|
||||
def _escape(word: str) -> str:
|
||||
"""A folded word holds no wildcard, but a caller may hand over raw text."""
|
||||
return re.sub(r"([\\%_])", r"\\\1", word)
|
||||
def known(token: str) -> bool:
|
||||
"""Whether the token names something: a country of the table (or `EU`), a
|
||||
region of `REGIONS`, a place with a value. `region:atlantis`, `country:ZZ`
|
||||
and `place:` are not."""
|
||||
return _known(parse(token))
|
||||
|
||||
|
||||
def _words(column: str, phrases: Iterable[str]) -> tuple[str, list]:
|
||||
"""Any of the phrases as whole words of the column. A multi-word phrase is
|
||||
a phrase: `kgs lyngby` needs both words, side by side."""
|
||||
phrases = [p for p in dict.fromkeys(phrases) if p]
|
||||
if not phrases:
|
||||
return _NEVER
|
||||
clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'"
|
||||
return " OR ".join([clause] * len(phrases)), [f"% {_escape(p)} %" for p in phrases]
|
||||
def _known(tok: Token) -> bool:
|
||||
if tok.kind == "country":
|
||||
return tok.value in _COUNTRY_LABELS or tok.value == "EU"
|
||||
if tok.kind == "region":
|
||||
return tok.value in REGIONS
|
||||
return bool(tok.value)
|
||||
|
||||
|
||||
def _country(code: str, place: str, country: str) -> tuple[str, list]:
|
||||
"""The role's own country, or a line that names the country while `country`
|
||||
is empty or `EU`: a board listing several offices has no single country."""
|
||||
names, params = _words(place, _COUNTRY_MENTIONS.get(code, ()))
|
||||
if not params:
|
||||
return f"{country} = ?", [code]
|
||||
return f"{country} = ? OR ({country} IN ('', 'EU') AND ({names}))", [code, *params]
|
||||
@dataclass(frozen=True)
|
||||
class _Matcher:
|
||||
"""What a token asks of a role. `phrases` are whole-word phrases of the
|
||||
folded place; `countries` are `(code, mentions)`: the role's own country, or
|
||||
a line with one of the mentions while `country` is empty or `EU`, since a
|
||||
board listing several offices has no single country."""
|
||||
|
||||
phrases: frozenset[str]
|
||||
countries: tuple[tuple[str, frozenset[str]], ...]
|
||||
longest: int
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def _matcher(token: str) -> _Matcher | None:
|
||||
"""The token read once: `None` when it names nothing.
|
||||
|
||||
A `place:` value that is also a region or a country is matched as that too:
|
||||
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
|
||||
written as a bare word means what the owner meant by it."""
|
||||
tok = parse(token)
|
||||
if not _known(tok):
|
||||
return None
|
||||
phrases: set[str] = set()
|
||||
countries: list[tuple[str, frozenset[str]]] = []
|
||||
if tok.kind == "country":
|
||||
countries.append((tok.value, frozenset(_COUNTRY_MENTIONS.get(tok.value, ()))))
|
||||
elif tok.kind == "region":
|
||||
phrases.update(_PHRASES[tok.value])
|
||||
else:
|
||||
phrases.update(_expand(tok.value))
|
||||
phrases.update(_PHRASES.get(_REGION_BY_NAME.get(tok.value, ""), ()))
|
||||
if code := _COUNTRY_BY_NAME.get(tok.value):
|
||||
countries.append((code, frozenset(_COUNTRY_MENTIONS.get(code, ()))))
|
||||
phrases.discard("")
|
||||
longest = max((len(p.split()) for p in (*phrases, *(m for _, ms in countries for m in ms))),
|
||||
default=1)
|
||||
return _Matcher(frozenset(phrases), tuple(countries), longest)
|
||||
|
||||
|
||||
@lru_cache(maxsize=32768)
|
||||
def _runs(place_fold: str, longest: int) -> frozenset[str]:
|
||||
"""Every run of up to `longest` words of a folded line, each a whole-word
|
||||
phrase the line holds."""
|
||||
words = place_fold.split()
|
||||
return frozenset(" ".join(words[i:i + n])
|
||||
for n in range(1, longest + 1) for i in range(len(words) - n + 1))
|
||||
|
||||
|
||||
def matches(place_fold: str, country: str, token: str) -> bool:
|
||||
"""Whether a role whose folded place and country are these is found by the
|
||||
token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or
|
||||
a Danish city) while `country` is empty or `EU`. `region:skane` is any of
|
||||
its names or towns as whole words, `place:malmo` the town and its aliases.
|
||||
A token that names nothing (`known` is false) matches no role."""
|
||||
match = _matcher(token)
|
||||
if match is None:
|
||||
return False
|
||||
runs = None
|
||||
if match.phrases:
|
||||
runs = _runs(place_fold, match.longest)
|
||||
if not match.phrases.isdisjoint(runs):
|
||||
return True
|
||||
for code, mentions in match.countries:
|
||||
if country == code:
|
||||
return True
|
||||
if mentions and country in ("", "EU"):
|
||||
runs = _runs(place_fold, match.longest) if runs is None else runs
|
||||
if not mentions.isdisjoint(runs):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _place_match(place_fold: str | None, country: str | None, token: str) -> int:
|
||||
return int(matches(place_fold or "", country or "", token))
|
||||
|
||||
|
||||
def register(conn: sqlite3.Connection) -> None:
|
||||
"""Give a connection `place_match(place_fold, country, token)`, which is 1 when
|
||||
`matches` is true. `match_sql` returns a clause that calls it."""
|
||||
conn.create_function(SQL_FUNCTION, 3, _place_match, deterministic=True)
|
||||
|
||||
|
||||
def match_sql(token: str, *, place: str = "place_fold", country: str = "country") \
|
||||
-> tuple[str, list]:
|
||||
"""SQL over the two columns for one token, parenthesised, with `?` parameters.
|
||||
Several tokens are joined with OR by the caller. Bare text is a place.
|
||||
|
||||
A `place:` value that is also a region or a country is matched as that too:
|
||||
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
|
||||
written as a bare word means what the owner meant by it. A token that names
|
||||
nothing known (`region:atlantis`, an empty value) matches no role."""
|
||||
"""SQL over the two columns for one token, with one `?` parameter. It calls
|
||||
`place_match`, so the connection needs `register`. Several tokens are joined
|
||||
with OR by the caller. Bare text is a place. A token that names nothing
|
||||
(`known` is false) is `0`, which matches no role."""
|
||||
tok = parse(token)
|
||||
if tok.kind == "country":
|
||||
sql, params = _country(tok.value, place, country)
|
||||
elif tok.kind == "region":
|
||||
sql, params = _words(place, _PHRASES.get(tok.value, ()))
|
||||
else:
|
||||
region = _REGION_BY_NAME.get(tok.value)
|
||||
sql, params = _words(place, [*_expand(tok.value), *_PHRASES.get(region, ())]) \
|
||||
if tok.value else _NEVER
|
||||
if tok.value in _COUNTRY_BY_NAME:
|
||||
more, extra = _country(_COUNTRY_BY_NAME[tok.value], place, country)
|
||||
sql, params = f"{sql} OR {more}", [*params, *extra]
|
||||
return f"({sql})", params
|
||||
if not _known(tok):
|
||||
return "0", []
|
||||
return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- suggestions
|
||||
@@ -442,9 +531,9 @@ def _prefixes(q: str, candidates: Iterable[str]) -> bool:
|
||||
return any(c.startswith(q) or f" {q}" in f" {c}" for c in candidates)
|
||||
|
||||
|
||||
#: Where one town ends in a line: `;` between places, and `or` / `eller` / `og`
|
||||
#: between two towns in one ("København V eller Aalborg").
|
||||
_PART = re.compile(r"\s*;\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
|
||||
#: Where one town ends in a line: `;` and `|` between places, and `or` / `eller` /
|
||||
#: `og` between two towns in one ("København V eller Aalborg").
|
||||
_PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
|
||||
|
||||
#: Danish boards append "and the possibility of working from home".
|
||||
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
|
||||
@@ -471,16 +560,32 @@ def _towns(place: str) -> list[str]:
|
||||
return found
|
||||
|
||||
|
||||
def _towns_in(folded: str) -> list[str]:
|
||||
"""The known towns a fold holds as whole words, longest first from the left,
|
||||
as their groups: `munich berlin` is Munich and Berlin, `copenhagen office` is
|
||||
Copenhagen, `hoje taastrup` is one town and not also Taastrup."""
|
||||
words = folded.split()
|
||||
found: list[str] = []
|
||||
i = 0
|
||||
while i < len(words):
|
||||
for n in range(min(_KNOWN_LONGEST, len(words) - i), 0, -1):
|
||||
if (group := _KNOWN.get(" ".join(words[i:i + n]))) is not None:
|
||||
found.append(group)
|
||||
i += n
|
||||
break
|
||||
else:
|
||||
i += 1
|
||||
return list(dict.fromkeys(found))
|
||||
|
||||
|
||||
def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[str, int]:
|
||||
"""Open roles per region, over folded lines. A line is cut into every run of
|
||||
words once, and each region asks whether it holds one of its phrases."""
|
||||
sets = {key: set(_PHRASES[key]) for key in keys}
|
||||
longest = max((len(p.split()) for key in sets for p in sets[key]), default=1)
|
||||
sets = {key: frozenset(_PHRASES[key]) for key in keys}
|
||||
longest = max((len(p.split()) for phrases in sets.values() for p in phrases), default=1)
|
||||
counts: Counter[str] = Counter()
|
||||
for text, roles in places:
|
||||
words = text.split()
|
||||
runs = {" ".join(words[i:i + n])
|
||||
for n in range(1, longest + 1) for i in range(len(words) - n + 1)}
|
||||
runs = _runs(text, longest)
|
||||
for key, phrases in sets.items():
|
||||
if not phrases.isdisjoint(runs):
|
||||
counts[key] += roles
|
||||
@@ -516,20 +621,32 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
|
||||
regions = [Suggestion(f"region:{key}", REGIONS[key].label, "region", counts[key])
|
||||
for key in named if counts[key]]
|
||||
|
||||
# A line's town is a group. A line that is a town is that group's spelling;
|
||||
# one that only holds a known town ("Copenhagen Office", "Munich - Berlin")
|
||||
# counts under each such town and shows as none; anything else is its own.
|
||||
counts_by_town: Counter[str] = Counter()
|
||||
spellings: dict[str, Counter[str]] = {}
|
||||
written: dict[str, set[str]] = {}
|
||||
for place, _, roles in rows:
|
||||
for town in _towns(place):
|
||||
folded = fold(town)
|
||||
group = _GROUPS[folded][0] if folded in _GROUPS else _variants(folded)[-1]
|
||||
if folded not in _KNOWN and (inside := _towns_in(folded)):
|
||||
for group in inside:
|
||||
counts_by_town[group] += roles
|
||||
continue
|
||||
group = _group(folded)
|
||||
counts_by_town[group] += roles
|
||||
spellings.setdefault(group, Counter())[town] += roles
|
||||
written.setdefault(group, set()).add(folded)
|
||||
towns = []
|
||||
for group, seen in spellings.items():
|
||||
if not _prefixes(text, [form for folded in written[group] for form in _expand(folded)]):
|
||||
for group, total in counts_by_town.items():
|
||||
forms = {group, *written.get(group, ())}
|
||||
if not _prefixes(text, [form for folded in forms for form in _expand(folded)]):
|
||||
continue
|
||||
shown = min(seen, key=lambda s: (-seen[s], s))
|
||||
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", sum(seen.values())))
|
||||
seen = spellings.get(group)
|
||||
shown = min(seen, key=lambda s: (-seen[s], s)) if seen \
|
||||
else _TOWN_LABELS.get(group) or group.title()
|
||||
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total))
|
||||
|
||||
countries.sort(key=lambda s: (-s.count, s.label))
|
||||
regions.sort(key=lambda s: (-s.count, s.label))
|
||||
|
||||
+766
-22
@@ -3,6 +3,7 @@
|
||||
The rows are `place` strings as boards write them, taken from the live database.
|
||||
Each token's SQL runs against an in-memory table shaped like `roles`."""
|
||||
|
||||
# ruff: noqa: RUF001 (real place strings, with the letters boards write)
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
@@ -10,7 +11,8 @@ import sqlite3
|
||||
import pytest
|
||||
|
||||
from app import places
|
||||
from app.places import REGIONS, Suggestion, Token, fold, label, match_sql, parse, suggest
|
||||
from app.places import (REGIONS, Suggestion, Token, fold, known, label, match_sql, matches,
|
||||
parse, register, suggest)
|
||||
|
||||
#: (place, country) as a board wrote them and `country_of` read them.
|
||||
ROWS = [
|
||||
@@ -37,9 +39,15 @@ ROWS = [
|
||||
]
|
||||
|
||||
|
||||
def connect() -> sqlite3.Connection:
|
||||
db = sqlite3.connect(":memory:")
|
||||
register(db)
|
||||
return db
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def table() -> sqlite3.Connection:
|
||||
db = sqlite3.connect(":memory:")
|
||||
db = connect()
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, ?)",
|
||||
[(place, fold(place), country) for place, country in ROWS])
|
||||
@@ -153,7 +161,7 @@ def test_copenhagen_is_every_spelling(table: sqlite3.Connection, spelling: str)
|
||||
|
||||
@pytest.mark.parametrize("group", places._ALIAS_GROUPS, ids=lambda g: g[0])
|
||||
def test_every_alias_finds_every_other(group: tuple[str, ...]) -> None:
|
||||
db = sqlite3.connect(":memory:")
|
||||
db = connect()
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, '')",
|
||||
[(name, fold(name)) for name in group])
|
||||
@@ -223,7 +231,7 @@ def test_a_country_is_also_a_line_that_names_it_when_country_is_unset(
|
||||
|
||||
|
||||
def test_a_country_name_does_not_reach_a_role_of_another_country() -> None:
|
||||
db = sqlite3.connect(":memory:")
|
||||
db = connect()
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, ?)", [
|
||||
("Remote, Denmark", "remote denmark", "GB"), ("Copenhagen", "copenhagen", "DE"),
|
||||
@@ -261,6 +269,15 @@ def test_jutland_and_funen(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "region:funen") == {"Odense, REGION_DNK 1852 South Denmark, Denmark"}
|
||||
|
||||
|
||||
def test_oresund_finds_a_line_of_either_region_but_is_named_by_its_own_words() -> None:
|
||||
assert matches("lund sweden", "SE", "region:oresund")
|
||||
assert matches("kobenhavn k", "DK", "region:oresund")
|
||||
assert not matches("aarhus", "DK", "region:oresund")
|
||||
assert "skane" not in {fold(n) for n in REGIONS["oresund"].names}
|
||||
assert REGIONS["oresund"].includes == ("hovedstaden", "skane")
|
||||
assert parse("region:Capital Region of Denmark").value == "hovedstaden"
|
||||
|
||||
|
||||
def test_oresund_is_the_two_capital_regions(table: sqlite3.Connection) -> None:
|
||||
assert found(table, "region:oresund") == (
|
||||
found(table, "region:hovedstaden") | found(table, "region:skane"))
|
||||
@@ -279,27 +296,28 @@ def test_a_token_that_names_nothing_finds_nothing(table: sqlite3.Connection) ->
|
||||
assert found(table, "place:nowhere-in-particular") == set()
|
||||
|
||||
|
||||
def test_the_columns_are_the_callers(table: sqlite3.Connection) -> None:
|
||||
def test_the_columns_are_the_callers() -> None:
|
||||
sql, params = match_sql("country:DK", place="r.pf", country="r.c")
|
||||
assert "r.pf" in sql and "r.c " in sql and "place_fold" not in sql
|
||||
db = sqlite3.connect(":memory:")
|
||||
assert sql == "place_match(r.pf, r.c, ?)" and params == ["country:DK"]
|
||||
db = connect()
|
||||
db.execute("CREATE TABLE r (pf TEXT, c TEXT)")
|
||||
db.executemany("INSERT INTO r VALUES (?, ?)", [("lund", "SE"), ("billund", "DK")])
|
||||
sql, params = match_sql("place:lund", place="r.pf", country="r.c")
|
||||
assert db.execute(f"SELECT pf FROM r WHERE {sql}", params).fetchall() == [("lund",)]
|
||||
|
||||
|
||||
def test_sql_is_parenthesised_and_has_one_parameter_per_question_mark() -> None:
|
||||
for token in ("country:DK", "region:oresund", "place:copenhagen", "place:denmark",
|
||||
"region:atlantis"):
|
||||
def test_sql_has_one_parameter_per_question_mark_and_a_token_is_normalised() -> None:
|
||||
for token in ("country:DK", "region:oresund", "place:copenhagen", "place:denmark", "Skåne"):
|
||||
sql, params = match_sql(token)
|
||||
assert sql.startswith("(") and sql.endswith(")")
|
||||
assert sql.count("?") == len(params), token
|
||||
assert sql.startswith("place_match(") and sql.count("?") == len(params) == 1, token
|
||||
assert match_sql("country:dk")[1] == ["country:DK"]
|
||||
assert match_sql("Kgs. Lyngby")[1] == ["place:kgs lyngby"]
|
||||
for token in ("region:atlantis", "country:ZZ", "place:", ""):
|
||||
assert match_sql(token) == ("0", []), token
|
||||
|
||||
|
||||
def test_a_wildcard_in_a_value_is_a_letter() -> None:
|
||||
assert places._escape("50%_off\\") == "50\\%\\_off\\\\"
|
||||
db = sqlite3.connect(":memory:")
|
||||
db = connect()
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, '')", [("a b", "a b"), ("axb", "axb"),
|
||||
("a_b", "a_b")])
|
||||
@@ -307,9 +325,26 @@ def test_a_wildcard_in_a_value_is_a_letter() -> None:
|
||||
sql, params = match_sql(token)
|
||||
assert {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)} \
|
||||
== {"a b"}, token
|
||||
# A column that is not folded still matches a literal underscore, not any character.
|
||||
sql, params = places._words("place", ["a_b"])
|
||||
assert db.execute(f"SELECT place FROM roles WHERE {sql}", params).fetchall() == [("a_b",)]
|
||||
|
||||
|
||||
def test_known_is_false_for_a_token_that_names_nothing() -> None:
|
||||
for token in ("region:atlantis", "country:ZZ", "place:", "region:", "country:", "", ":"):
|
||||
assert not known(token), token
|
||||
for token in ("country:DK", "country:EU", "country:dk", "region:skane", "region:Skåne",
|
||||
"place:nowhere-in-particular", "Skåne", "country:Denmark"):
|
||||
assert known(token), token
|
||||
|
||||
|
||||
def test_matches_is_false_for_a_token_that_names_nothing() -> None:
|
||||
assert not matches("copenhagen", "DK", "region:atlantis")
|
||||
assert not matches("copenhagen", "ZZ", "country:ZZ")
|
||||
assert not matches("copenhagen", "DK", "place:")
|
||||
|
||||
|
||||
def test_null_columns_match_nothing_and_do_not_raise() -> None:
|
||||
db = connect()
|
||||
assert db.execute("SELECT place_match(NULL, NULL, 'place:lund')").fetchone() == (0,)
|
||||
assert db.execute("SELECT place_match(NULL, 'DK', 'country:DK')").fetchone() == (1,)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- the tables
|
||||
@@ -318,7 +353,8 @@ def test_a_wildcard_in_a_value_is_a_letter() -> None:
|
||||
def test_every_region_has_names_and_towns() -> None:
|
||||
assert set(REGIONS) == {"skane", "hovedstaden", "zealand", "oresund", "jutland", "funen"}
|
||||
for key, region in REGIONS.items():
|
||||
assert region.key == key and region.label and region.names and region.towns
|
||||
assert region.key == key and region.label and region.names
|
||||
assert region.towns or region.includes
|
||||
|
||||
|
||||
def test_the_bare_word_zealand_is_no_name_since_new_zealand_has_it() -> None:
|
||||
@@ -348,6 +384,11 @@ PLACES = [
|
||||
("Remote", "", 40), ("Hybrid", "", 12), ("Europe", "EU", 20), ("", "", 5),
|
||||
("Berlin, Germany", "DE", 100), ("Auckland, New Zealand", "NZ", 25),
|
||||
("The Hague, Netherlands", "NL", 9), ("Den Haag", "NL", 4),
|
||||
("Copenhagen Office", "DK", 16), ("Copenhagen Hub - Løvstræde", "DK", 14),
|
||||
("DK-Copenhagen-MSO", "DK", 2),
|
||||
("Munich", "DE", 40), ("München", "DE", 10), ("Munich Office", "DE", 5),
|
||||
("Munich (DE)", "DE", 3), ("Munich - Berlin", "DE", 2),
|
||||
("Garching bei München (Munich)", "DE", 4), ("Kalundborg, Region Zealand, Denmark", "DK", 14),
|
||||
]
|
||||
|
||||
|
||||
@@ -365,7 +406,8 @@ def test_skane_is_suggested_however_it_is_typed() -> None:
|
||||
def test_a_country_counts_the_roles_that_carry_its_code() -> None:
|
||||
hit = suggest("den", PLACES)[0]
|
||||
assert hit == Suggestion("country:DK", "Denmark", "country",
|
||||
69 + 56 + 35 + 34 + 16 + 18 + 26 + 10 + 27 + 1)
|
||||
69 + 56 + 35 + 34 + 16 + 18 + 26 + 10 + 27 + 1
|
||||
+ 16 + 14 + 2 + 14)
|
||||
assert tokens("swe")[0] == "country:SE"
|
||||
assert tokens("sverige")[0] == "country:SE"
|
||||
assert tokens("danmark")[0] == "country:DK"
|
||||
@@ -394,7 +436,7 @@ def test_spellings_of_one_town_are_one_suggestion() -> None:
|
||||
towns = [s for s in suggest("cop", PLACES) if s.kind == "place"]
|
||||
assert len(towns) == 1
|
||||
# København, K and Ø, the one in a list of three towns; Copenhagen, and "Remote OR Copenhagen".
|
||||
assert towns[0].label == "København" and towns[0].count == 69 + 35 + 34 + 1 + 56 + 1
|
||||
assert towns[0].label == "København" and towns[0].count == 69 + 35 + 34 + 1 + 56 + 1 + 16 + 14 + 2
|
||||
assert [s.token for s in suggest("kobenhavn", PLACES) if s.kind == "place"] \
|
||||
== [towns[0].token]
|
||||
|
||||
@@ -425,9 +467,10 @@ def test_the_word_must_start_not_end() -> None:
|
||||
assert "place:albertslund" not in tokens("lund")
|
||||
|
||||
|
||||
def test_new_zealand_is_a_country_and_zealand_a_region_only_when_it_has_roles() -> None:
|
||||
def test_a_country_or_region_with_no_open_role_is_not_suggested() -> None:
|
||||
assert tokens("zea")[:1] == ["country:NZ"]
|
||||
assert "region:zealand" not in tokens("zea") # no Zealand role in PLACES
|
||||
assert "region:zealand" in tokens("zea") # Kalundborg
|
||||
assert "region:funen" not in tokens("fun") # no Funen role in PLACES
|
||||
assert "country:AU" not in tokens("aus") # no role of its own
|
||||
|
||||
|
||||
@@ -451,3 +494,704 @@ def test_a_suggested_token_finds_the_roles_it_counted(table: sqlite3.Connection)
|
||||
assert parse(s.token).kind == s.kind
|
||||
assert found(table, s.token), s.token
|
||||
assert label(s.token) == s.label or s.kind == "place"
|
||||
|
||||
|
||||
# ------------------------------------------------- the function against LIKE
|
||||
|
||||
#: `country<TAB>place` of open roles on the live database, distinct: a random
|
||||
#: sample plus, for each token below, lines that hold it. Some lines have no country.
|
||||
REAL = """
|
||||
NL 's-Hertogenbosch, Netherlands
|
||||
3 Locations
|
||||
5 Locations
|
||||
DK Aabenraa eller København
|
||||
DK Aalborg, Copenhagen, Kolding, Aarhus, Hybrid position
|
||||
DK Aalborg, Denmark; Aarhus, Denmark; Copenhagen, Denmark; Odense, Denmark; Søborg, Denmark
|
||||
DK Aarhus C, Denmark
|
||||
DK Aarhus N, Denmark
|
||||
DK Aarhus or Valby
|
||||
EU Aarhus, Denmark; København, Denmark; Stockholm, Sweden
|
||||
Aix-en-Provence
|
||||
DK Allerød or Stockholm (SE)
|
||||
DK Allerød, Copenhagen, or Stockholm, Hybrid position
|
||||
KZ Almaty, Almatı oblısı, Kazakhstan
|
||||
EU Amsterdam, Netherlands; Berlin, Germany; København, Denmark; Vilnius, Lithuania
|
||||
EU Amsterdam, Netherlands; Berlin, Germany; Munich, Germany; Remote, Germany
|
||||
EU Amsterdam, Netherlands; Manchester, United Kingdom
|
||||
EU Amsterdam; Germany
|
||||
FR Angers, France
|
||||
DE Ansbach, Mittelfranken, Germany
|
||||
MX Apaseo el Grande, GUANAJUATO, Mexico
|
||||
US Arizona; California; Washington
|
||||
NL Arnhem, GELDERLAND, Netherlands
|
||||
BE Asse, , Belgium
|
||||
Athens, Greece; Cyprus; Frankfurt am Main, Germany; Munich, Germany; Tel Aviv, Israel
|
||||
Athens, Greece; London, United Kingdom; Tel Aviv, Israel
|
||||
Auckland
|
||||
US Austin; New York City; Palo Alto
|
||||
AU Australian Capital Territory, Australia
|
||||
DK Bad Homburg; Copenhagen
|
||||
DK Bagsværd
|
||||
DK Ballerup or Aarhus V
|
||||
DK Ballerup, , Denmark
|
||||
DK Ballerup, Ballerup
|
||||
DK Ballerup, Denmark
|
||||
IN Bangalore, KA, India
|
||||
CN Bao Ding, Hebei, China
|
||||
ES Barcelona, Catalonia
|
||||
EU Barcelona, Spain; Stockholm, Stockholm County, Sweden
|
||||
EU Barcelona; Berlin; London; Munich; Paris
|
||||
EU Barcelona; Geneva
|
||||
DE Barsinghausen, Germany
|
||||
PL Bartoszyce, Warminsko-mazurskie, Poland
|
||||
MY Bayan lepas, Pulau Pinang, Malaysia
|
||||
EU Belgium; Netherlands
|
||||
RS Belgrade - Savski Venac, Serbia
|
||||
EU Belgrade, Serbia; Berlin, Germany; Munich, Germany
|
||||
US Bellevue, United States of America
|
||||
US Bellevue, WA, United States
|
||||
US Bellevue, Washington; Denver, Colorado; Seattle, Washington; Washington, D.C.
|
||||
IN Bengaluru, India
|
||||
RS Beograd, , Serbia
|
||||
NO Bergen, , Norway
|
||||
NL Berkel en Rodenrijs, ROTTERDAM, Netherlands
|
||||
DE Berlin Office; Munich Office
|
||||
DE Berlin Office; Munich Office; Remotely in Germany
|
||||
DE Berlin, Berlin, Germany
|
||||
DE Berlin, Germany
|
||||
EU Berlin, Germany; Amsterdam, Netherlands; Copenhagen, Denmark
|
||||
EU Berlin, Germany; Copenhagen, Denmark
|
||||
EU Berlin, Germany; London, United Kingdom
|
||||
EU Berlin, Germany; London, United Kingdom; Stockholm, Sweden
|
||||
EU Berlin, Germany; Munich, Germany; Prague, Czech Republic
|
||||
EU Berlin, Germany; Paris, France
|
||||
DE Berlin; Cologne; Frankfurt; Hamburg; Munich; Stuttgart
|
||||
DE Berlin; Frankfurt; Munich
|
||||
DE Berlin; Frankfurt; Munich; Stuttgart
|
||||
EU Berlin; London; Munich
|
||||
EU Berlin; London; Munich; Paris
|
||||
EU Berlin; London; Munich; Paris; Stockholm; Tallinn
|
||||
DE Berlin; Munich
|
||||
CH Bern, Switzerland
|
||||
NL Best, Noord-Brabant, Netherlands
|
||||
DE Bietigheim-Bissingen, Baden-Württemberg, Germany
|
||||
DK Billund Kornmarken, Denmark
|
||||
DK Billund, Denmark
|
||||
DK Billund, København
|
||||
DK Billund, København K
|
||||
DK Billund; Copenhagen Hub - Løvstræde
|
||||
EU Billund; London; Copenhagen Hub - Løvstræde
|
||||
DK Birkerød
|
||||
SE Bjuv
|
||||
DK Bjæverskov
|
||||
DE Bochum, NW, Germany
|
||||
DE Bonn, Germany
|
||||
FR Bordeaux
|
||||
JP Bosch Corporation_Shibuya_BGSJ, Tokyo, Japan
|
||||
US Boston, MA, United States
|
||||
US Bozeman, MT, United States
|
||||
DK Brabrand, Denmark
|
||||
Brest
|
||||
US Bridgewater, United States of America
|
||||
FR Briey, France
|
||||
GB Bristol, UK; London, UK
|
||||
SI Brnik - Aerodrom, , Slovenia
|
||||
DK Bronshoj, Denmark
|
||||
BE Bruxelles, Bruxelles, Belgium
|
||||
DK Brøndby
|
||||
DK Brøndby eller Fredericia
|
||||
DK Brøndby eller Fredericia og mulighed for hjemmearbejde
|
||||
US Burlington, United States
|
||||
Busan
|
||||
DE Butzbach, HE, Germany
|
||||
RO Buzau, Com Caragele, Constanta, Romania
|
||||
SE Båstad
|
||||
DE Böblingen, Baden-Württemberg, Germany
|
||||
CA CA-Toronto
|
||||
AU Camberwell, Australia
|
||||
GB Carlisle, England, United Kingdom
|
||||
DE Carlsfelder Str. 3, 06188 Land, Germany
|
||||
DE Castroper Hellweg 109, 44805 B, Germany
|
||||
MX Celaya Planta, GUANAJUATO, Mexico
|
||||
US Cement, United States
|
||||
Chandigarh
|
||||
CN Chengdu, Sichuan, China; XiAn, Shaanxi, China
|
||||
FR Cherbourg-en-Cotentin, Manche, France
|
||||
US Chicago, Atlanta, US-Remote, Toronto
|
||||
CL Chile
|
||||
China - Hong Kong; Taiwan - Taipei
|
||||
MX Ciudad de México, CDMX, Mexico
|
||||
MX Colon, QUERETARO, Mexico
|
||||
DK Copenhagen
|
||||
DK Copenhagen Hub - Løvstræde, Denmark
|
||||
DK Copenhagen Hub - Løvstræde; Billund
|
||||
DK Copenhagen Hub - Østergade, Denmark
|
||||
DK Copenhagen K, Denmark; Aarhus C, Denmark
|
||||
DK Copenhagen O, Denmark
|
||||
DK Copenhagen Office
|
||||
DK Copenhagen S, Region Capital of Denmark, Denmark
|
||||
DK Copenhagen V, Denmark
|
||||
DK Copenhagen V, Denmark; Aarhus C, Denmark
|
||||
DK Copenhagen or Aarhus
|
||||
Copenhagen, Denmark; Tel Aviv, Israel
|
||||
DK Copenhagen, Hovedstaden, Denmark
|
||||
DK Copenhagen; Billund LEGO Campus
|
||||
Copenhagen; Zurich; Singapore; Warsaw; Toronto
|
||||
AR Corrientes, , Argentina
|
||||
Corte Madera
|
||||
NL Culemborg, Netherlands
|
||||
Curitiba
|
||||
PE Cusco, Cusco, Peru
|
||||
DE DE Store Hamburg Spitalerstraße, Germany
|
||||
US Dallas, United States
|
||||
AU Dandenong South, VICTORIA, Australia
|
||||
GB Darlington, England, United Kingdom
|
||||
AU Deakin ACT, Australia
|
||||
EU Denmark; Vilnius, Lithuania
|
||||
DK Dianalund
|
||||
CN Dongguan, Guangdong, China
|
||||
DE Dortmund, Germany
|
||||
DE Dortmund, North Rhineland Westphalia, DE +1 more…
|
||||
IE Dublin, CA (ZICC)
|
||||
EU Dublin, Ireland; London
|
||||
GB Dublin, United Kingdom
|
||||
Dunkerque
|
||||
Durban
|
||||
EU EMEA; Germany; London, United Kingdom; Paris, France; Remote - Netherlands
|
||||
DE Ergolding, BY, Germany
|
||||
DK Esbjerg, Region South Denmark, Denmark
|
||||
DE Eschborn, HE, Germany
|
||||
DE Eschenburg, HE, Germany
|
||||
Ezhou
|
||||
IT Fagagna, , Italy
|
||||
GB Farnborough, Hampshire, United Kingdom
|
||||
DK Farum
|
||||
DK Finland, København, Litauen, Polen, Sverige
|
||||
EU Finland; France; Germany; Netherlands; Remote - Europe; Sweden; United Kingdom
|
||||
EU Finland; Remote - Denmark; Stockholm, Sweden
|
||||
Finland; United Kingdom; Korea, Republic of; Denmark; France; India; Poland; United States
|
||||
Finland; United Kingdom; United States; Belgium
|
||||
DE Fins Only-DE-Munich-MSO
|
||||
EU Fleet Place Office; Amsterdam Office; Berlin Office
|
||||
France; India; Finland
|
||||
DE Frankfurt; Berlin; Munich
|
||||
DE Frankfurt; Munich
|
||||
DK Fredericia or Ballerup
|
||||
DK Fredericia, REGION_DNK 1852 South Denmark, Denmark
|
||||
DK Frederiksberg
|
||||
DK Frederikssund og mulighed for hjemmearbejde
|
||||
DK Fårevejle
|
||||
GB GB-London
|
||||
US Garden City, TX, United States
|
||||
EU Geneva; London
|
||||
DE Gerlingen, Baden-Württemberg, Germany
|
||||
DE Germany Scoober Office
|
||||
EU Germany, Remote; The Netherlands, Remote
|
||||
Germany; Portugal; Canada; United States
|
||||
DK Glostrup
|
||||
DK Glostrup, Denmark
|
||||
PL Goleniów, Województwo zachodniopomorskie, Poland
|
||||
US Greenville, South Carolina, United States, South Carolina, United States of America
|
||||
PL Grodzisk Mazowiecki (mazowieckie), Poland
|
||||
SE Göteborg, Västra Götaland, Sweden; Lund, Skåne, Sweden
|
||||
AT Haag, Austria
|
||||
DE Hannover, NDS, Germany
|
||||
VN Hanoi, , Vietnam
|
||||
DK Headquarters, Denmark
|
||||
US Heber, AZ, United States
|
||||
SE Helsingborg, Sweden
|
||||
FI Helsinki Metropolitan Area
|
||||
FI Helsinki, Finland
|
||||
EU Helsinki, Finland; Stockholm, Sweden; Tallinn, Estonia
|
||||
DK Herlev og mulighed for hjemmearbejde
|
||||
DE Herzogenrath, North Rhine Westphalia, Germany
|
||||
DE Hildesheim, NDS, Germany
|
||||
DK Hillerød
|
||||
DK Hillerød, Capital Region of Denmark, Denmark
|
||||
DE Hoher Hagen 3, 58513 Lüdensche, Germany
|
||||
DK Holbæk
|
||||
DE Homburg, , Germany
|
||||
DE Homburg/Saar, SL, Germany
|
||||
EU Home Based - Americas; Home Based - APAC; Home based - EMEA; Home based - Worldwide
|
||||
GB Home Based - Americas; Home based - EMEA; Office Based - London, UK
|
||||
MX Home-based, Mexico
|
||||
DK Hvalsø
|
||||
DK Hvidovre
|
||||
Hwaseong-si
|
||||
Hybrid
|
||||
GB Hybrid - London, England
|
||||
IN Hyderabad; Noida
|
||||
SE Härryda, Sweden
|
||||
SE Hässleholm
|
||||
SE Höör
|
||||
JP Ibaraki, Osaka, Japan
|
||||
US Illinois, USA, Remote; Minnesota, USA, Remote
|
||||
DE Immenstadt im Allgäu, BY, Germany
|
||||
Indija
|
||||
IT Induno Olona, Italy
|
||||
Israel; Remote - Europe; United Kingdom
|
||||
DE Jacob-A.-Morand-Str. 18, 07552, Germany
|
||||
JP Japan, Tokyo
|
||||
Jena
|
||||
CZ Jihlava 1, Kraj Vysočina, Czechia
|
||||
NL Joure, Netherlands
|
||||
DK Kalundborg, Region Zealand, Denmark
|
||||
DK Kastrup
|
||||
SE Katrineholm, Sweden
|
||||
DE Kelsterbach, HE, Germany
|
||||
Kempten im Allgäu
|
||||
IL Kfar Saba, Israel
|
||||
Konstanz
|
||||
SE Kristianstad, Sweden
|
||||
MY Kuala Lumpur – Co-labs, Kuala Lumpur – Co-labs, Malaysia
|
||||
MY Kuala Lumpur, Malaysia
|
||||
SE Kungsängen, Sweden
|
||||
BE Kuurne, Belgium
|
||||
DK København
|
||||
DK København K
|
||||
DK København K eller Odense C
|
||||
DK København K og mulighed for hjemmearbejde
|
||||
DK København NV
|
||||
DK København S
|
||||
DK København SV eller Aalborg og mulighed for hjemmearbejde
|
||||
DK København SV, Odense C
|
||||
DK København V eller Korsør
|
||||
DK København V og mulighed for hjemmearbejde
|
||||
DK København Ø og Frederiksberg C samt mulighed for hjemmearbejde
|
||||
DK København Ø, Denmark
|
||||
EU København, Capital RegionDenmark, Denmark; Moss, Østfold, Norway; Oslo, Oslo, Norway
|
||||
DK København, Denmark
|
||||
DK København, Denmark; Aarhus, Denmark; Aalborg, Denmark; Kolding, Denmark
|
||||
DK København, Denmark; Aarhus, Denmark; Kolding, Denmark; Aalborg, Denmark
|
||||
EU København, Denmark; Stockholm, Sweden
|
||||
DK København, København SV
|
||||
DK København, København SV, Odense, Odense C
|
||||
DK København, København SV, Viby J, Aarhus
|
||||
DK København, Roskilde
|
||||
DK Køge
|
||||
DK Køge, Denmark
|
||||
ES La Muela (Zaragoza), Z, Spain
|
||||
La Roche-sur-Yon
|
||||
US Lake Zurich, Illinois, United States
|
||||
SE Landskrona
|
||||
EU Lausanne; London
|
||||
DK Lem St, Denmark
|
||||
DK Lem St, Region Central Jutland, Denmark
|
||||
NL Lemelerveld, Netherlands
|
||||
MX León, GUA, Mexico
|
||||
IT Liguria, Italy
|
||||
DE Lohr am Main, BY, Germany
|
||||
SE Lomma
|
||||
London, Canada; Remote, United States; Oak Brook, United States
|
||||
GB London, London, United Kingdom
|
||||
London, Paris, Toronto, New York
|
||||
EU London, United Kingdom; Oslo, Norway
|
||||
London, United Kingdom; Remote, United States
|
||||
GB London; Billund Project House
|
||||
EU London; Copenhagen
|
||||
GB London; Remote (UK)
|
||||
EU London; Warsaw; Copenhagen
|
||||
EU London; Zurich
|
||||
SE Luleå, Sweden
|
||||
SE Lund, Skåne län, Sweden
|
||||
SE Lund, Skåne, Sweden
|
||||
US Madison, Wisconsin, United States of America
|
||||
SE Malmo, Sweden
|
||||
SE Malmö
|
||||
SE Malmö, Skåne County, Sweden
|
||||
SE Malmö, Skåne Län, Sweden
|
||||
SE Malmö, Sweden
|
||||
DK Maribo
|
||||
DE Meschede, Nordrhein-Westfalen, Germany
|
||||
Metz
|
||||
US Miami
|
||||
US Miami, Florida, United States
|
||||
DK Middelfart, Denmark
|
||||
AU Midland, Australia
|
||||
IT Milan Office
|
||||
EU Milan, Italy; Paris, Paris, France
|
||||
US Minneapolis, Minnesota, United States, Minnesota, United States of America
|
||||
US Minneola, KS, United States
|
||||
Monterrey
|
||||
MX Monterrey, Mexico
|
||||
US Mountain View, CA, USA; San Francisco, CA, USA
|
||||
US Mountain View, California
|
||||
US Mountain View, California; Plano, Texas; San Francisco, California
|
||||
DE Munich
|
||||
EU Munich - Berlin - London - Paris
|
||||
EU Munich - Berlin - London - Paris - Warsaw - Tallinn
|
||||
EU Munich - Berlin - Paris
|
||||
DE Munich, Bavaria, Germany
|
||||
DK Munkebo, Denmark
|
||||
LU Munsbach, Luxembourg
|
||||
Mátészalka
|
||||
Möhnesee
|
||||
DE München
|
||||
DE München (DE)
|
||||
DE München, BAYERN, Germany
|
||||
DE München, BY, Germany
|
||||
DE München, Germany
|
||||
US NYC
|
||||
US NYC, SF, Chi, Remote
|
||||
Nagpur
|
||||
DK Nakskov, Denmark
|
||||
DK Nakskov, Region Zealand, Denmark
|
||||
Nanjing
|
||||
JP Nasushiobara, Tochigi, Japan
|
||||
GB New Britain
|
||||
New York City, Toronto, Chicago, or Remote
|
||||
US New York, San Francisco, Seattle, Atlanta, Chicago
|
||||
New York; Hong Kong; London; Singapore; Sydney
|
||||
RS Niš, Serbia
|
||||
DK Nordhavn eller Aalborg og mulighed for hjemmearbejde
|
||||
DK Nordhavn eller Viby J
|
||||
DK Nordhavn or Åbyhøj
|
||||
NO Norway
|
||||
AU Nundah, Australia
|
||||
SE Nybro, Sweden
|
||||
DK Nykøbing F
|
||||
DK Nykøbing F, Denmark
|
||||
DK Næstved
|
||||
DK Nørre Asmindrup
|
||||
DE Nürnberg
|
||||
CA ON, Toronto, Canada
|
||||
DE Oberkochen; München
|
||||
PL Oborniki, Poland
|
||||
DK Odense C, Denmark
|
||||
DK Odense M, Denmark
|
||||
DK Odense, Denmark
|
||||
DK Odense, REGION_DNK 1852 South Denmark, Denmark
|
||||
GB Office Based - London, UK
|
||||
TW Office Based - Taipei, Taiwan
|
||||
US Oklahoma, United States
|
||||
CA Ontario - Remote
|
||||
US Ontario, CA, United States of America
|
||||
US Orange County , California, United States
|
||||
SE Oskarshamn, Sweden
|
||||
EU Paris, France; Copenhagen, Denmark; Amsterdam, Netherlands; Berlin, Germany
|
||||
FR Paris, France; Tassin-la-Demi-Lune, France; Bordeaux, France
|
||||
EU Paris;Geneva
|
||||
US Paw Paw, IL, United States
|
||||
FR Perigny, France
|
||||
FR Perpignan, Pyrénées-Orientales, France
|
||||
SE Perstorp
|
||||
US Phillipsburg, New Jersey, United States
|
||||
DE Pinkertweg 16, 22113 Hamburg, Germany
|
||||
MX Planta Celaya, , Mexico
|
||||
EU Poland; Hungary
|
||||
PT Porto, Portugal
|
||||
EU Porto, Portugal; Madrid, Spain; Berlin, Germany
|
||||
CZ Prague
|
||||
EC Quito, , Ecuador
|
||||
DK Randers SV, København V or Krakow
|
||||
GB Redhill (GB)
|
||||
DK Region Hovedstaden
|
||||
DK Region Sjælland
|
||||
IL Rehovot, Israel
|
||||
AU Remote - Australia
|
||||
CA Remote - CAN British Columbia
|
||||
Remote - Portugal; United States
|
||||
US Remote - USA WA, United States of America
|
||||
US Remote - United States
|
||||
DK Remote OR Copenhagen
|
||||
US Remote US MD, United States of America
|
||||
Remote job
|
||||
Remote, AMER
|
||||
Remote, Canada; Remote, Poland; Remote, United Kingdom; Remote, United States
|
||||
Remote, Canada; Remote, United Kingdom; Remote, US
|
||||
EU Remote, Germany; Dublin, Ireland; Lisbon, Portugal
|
||||
US Remote, US, Georgia
|
||||
US Remote, Washington, USA, United States of America
|
||||
BG Remote- Bulgaria
|
||||
Remote-NORAM
|
||||
US Remote-US
|
||||
RS Remote; Serbia
|
||||
DE Remscheid
|
||||
CA Retail Store Chinook Ctr, Canada
|
||||
US Retail Store Ontario Mills, United States of America
|
||||
HR Rijeka, Croatia
|
||||
DK Ringkøbing, Denmark
|
||||
DK Ringsted
|
||||
DK Ringsted, Ringsted
|
||||
BR Rio de Janeiro, Brasil
|
||||
DE Rosenheim
|
||||
DK Roskilde
|
||||
DK Roskilde og mulighed for hjemmearbejde
|
||||
DK Roskilde, Denmark
|
||||
PL Rosówek, Poland
|
||||
Roßdorf
|
||||
AU Ryde, Australia
|
||||
DE Röntgenstr. 33, 32107 Bad Salz, Germany
|
||||
DK Rønne or Køge
|
||||
Saint-Etienne
|
||||
Saint-Malo
|
||||
US San Bernardino, California, United States, California, United States of America
|
||||
US San Francisco, CA; Washington, DC
|
||||
US San Francisco, Seattle, New York, Chicago, Remote in the US
|
||||
CR San Jose, Costa Rica
|
||||
HN San Pedro Sula, , Honduras
|
||||
MT San Ġwann, Malta
|
||||
DE Schwäbisch Gmünd, BW, Germany
|
||||
US Seattle, Washington, United States
|
||||
US Seattle, Washington, United States, Washington, United States of America
|
||||
US Seymour, United States
|
||||
CN Shanghai, Shanghai, China; Suzhou, Jiangsu, China
|
||||
PL Siedlc, Poland
|
||||
SE Sigtuna, Sweden
|
||||
SG Singapore - Singapore
|
||||
SG Singapore, SG
|
||||
DK Slagelse
|
||||
SE Sollentuna, Sweden
|
||||
SE Solna
|
||||
BR Sorocaba, SP, Brazil
|
||||
DK Sorø
|
||||
GB Southampton, England, United Kingdom
|
||||
SE Staffanstorp
|
||||
US Stamford, Connecticut, United States, Connecticut, United States of America
|
||||
SE Stockholm HQ
|
||||
SE Stockholm, Stockholm County, Sweden
|
||||
EU Stockholm, Stockholm County, Sweden; London, London, United Kingdom; Barcelona, Spain
|
||||
SE Stockholm, Stockholm County, Sweden; Malmö, Sweden
|
||||
EU Stockholm, Sweden; Berlin, Germany
|
||||
EU Stockholm, Sweden; Jorvas, Kirkkonummi, Finland
|
||||
EU Stockholm; London; Milan; Warsaw
|
||||
CZ Strakonice, Czechia
|
||||
SE Svedala
|
||||
SE Sweden
|
||||
SE Sweden - Linköping
|
||||
SE Sweden - Lund
|
||||
GB Swindon, United Kingdom
|
||||
AU Sydney Or Melbourne
|
||||
US Syracuse, New York, United States, New York, United States of America
|
||||
PT São João, Aveiro, Portugal
|
||||
SE Södertälje, Sweden
|
||||
DK Taastrup
|
||||
DK Taastrup og mulighed for hjemmearbejde
|
||||
TW Taichung, Taichung City, Taiwan
|
||||
TW Taipei, Taiwan, Taiwan
|
||||
Taipei; Geneva
|
||||
CN Taiyuan, Shanxi, China
|
||||
EE Tallinn, Estonia
|
||||
GE Tbilisi, Georgia
|
||||
MX Tecnoparque, Mexico City, Mexico; Santa Fe, Mexico City, Mexico; Mexico City, Mexico
|
||||
MX Tepotzotlan, Estado de México, Mexico
|
||||
US The Woodlands, United States of America
|
||||
RO Timisoara, Romania
|
||||
GB Tiverton, United Kingdom
|
||||
SE Tomelilla
|
||||
MX Tonalá, Mexico
|
||||
Tonsley SA
|
||||
Toronto, New York, San Francisco, Atlanta, Seattle
|
||||
PL Toruń, PM, Poland
|
||||
PL Toruń, Poland
|
||||
SE Trelleborg
|
||||
Troisdorf
|
||||
NO Trondheim, Norway
|
||||
Tunis
|
||||
DE Tübingen (DE)
|
||||
US US-IL-Chicago-MSO
|
||||
USA (Remote), Spain (Remote), UK (Remote)
|
||||
US USA - IL - Chicago, United States of America
|
||||
US USA - UT - Remote, United States of America
|
||||
GB United Kingdom Remote
|
||||
EU United Kingdom; Paris, Ile-de-France, France
|
||||
United States; Stockholm, Sweden; Melbourne, Victoria, Australia
|
||||
United States; Sweden
|
||||
CN Urumqi, Xinjiang, China
|
||||
FR Valbonne, Provence-Alpes-Côte d'Azur, France
|
||||
DK Valby, Aalborg
|
||||
DK Vallensbæk Strand
|
||||
CA Vancouver
|
||||
Vannes
|
||||
FI Vantaa, Uusimaa, Finland
|
||||
SE Varberg, Sweden
|
||||
SE Vellinge
|
||||
US Vernon Hills, IL, United States
|
||||
DK Vi søger i hele landet
|
||||
US Victorville, California, United States
|
||||
DK Videbæk, Denmark
|
||||
DK Videbæk, Region Central Jutland, Denmark
|
||||
EU Vilnius, Lithuania; Copenhagen K, Denmark
|
||||
EU Vilnius, Lithuania; Copenhagen V, Denmark
|
||||
DK Værløse
|
||||
EU Warsaw, Poland; Vilnius, Lithuania
|
||||
PL Warszawa
|
||||
AU Wensleydale, VIC, Australia
|
||||
DE Wiesbaden, HE, Germany
|
||||
CA Windsor, Canada
|
||||
US Windsor, United States
|
||||
GB Worksop, UK
|
||||
Wuhan
|
||||
CN XiAn, Shaanxi, China
|
||||
SE Ystad
|
||||
CN Zhuhai, Guangdong, China
|
||||
SE Älmhult, Sweden
|
||||
SE Ängelholm
|
||||
SE Ängelholm, Sweden
|
||||
SE Åby, Sweden
|
||||
DK Åbyhøj eller Ballerup
|
||||
SE Örebro, Sweden
|
||||
SE Östra Göinge
|
||||
CN 东莞, , China
|
||||
"""
|
||||
|
||||
LINES = [tuple(reversed(line.split("\t", 1))) for line in REAL.strip("\n").split("\n")]
|
||||
|
||||
#: The tokens of every kind that are known. (`country:ZZ` and the rest that name
|
||||
#: nothing are `known` false and match nothing, which the old LIKE did not say.)
|
||||
ORACLE_TOKENS = [
|
||||
"country:DK", "country:SE", "country:DE", "country:GB", "country:US", "country:EU",
|
||||
"country:NO", "country:CH",
|
||||
"region:skane", "region:hovedstaden", "region:zealand", "region:oresund",
|
||||
"region:jutland", "region:funen",
|
||||
"place:copenhagen", "place:kobenhavn", "place:malmo", "place:lund", "place:munich",
|
||||
"place:zurich", "place:london", "place:remote", "place:aarhus", "place:kgs lyngby",
|
||||
"place:the hague", "place:scania", "place:capital region of denmark",
|
||||
"place:denmark", "place:germany", "place:skane", "Skåne", "Berlin",
|
||||
]
|
||||
|
||||
|
||||
def like_words(column: str, phrases) -> tuple[str, list]:
|
||||
"""The clause `match_sql` was before it called a function: each phrase a
|
||||
`LIKE` over the padded column. Slow, and the definition `matches` must keep."""
|
||||
phrases = [p for p in dict.fromkeys(phrases) if p]
|
||||
clause = f"' ' || {column} || ' ' LIKE ? ESCAPE '\\'"
|
||||
return " OR ".join([clause] * len(phrases)) or "0", [f"% {p} %" for p in phrases]
|
||||
|
||||
|
||||
def like_country(code: str) -> tuple[str, list]:
|
||||
names, params = like_words("place_fold", places._COUNTRY_MENTIONS.get(code, ()))
|
||||
if not params:
|
||||
return "country = ?", [code]
|
||||
return f"country = ? OR (country IN ('', 'EU') AND ({names}))", [code, *params]
|
||||
|
||||
|
||||
def like_sql(token: str) -> tuple[str, list]:
|
||||
tok = parse(token)
|
||||
if tok.kind == "country":
|
||||
sql, params = like_country(tok.value)
|
||||
elif tok.kind == "region":
|
||||
sql, params = like_words("place_fold", places._PHRASES[tok.value])
|
||||
else:
|
||||
region = places._REGION_BY_NAME.get(tok.value, "")
|
||||
sql, params = like_words("place_fold", [*places._expand(tok.value),
|
||||
*places._PHRASES.get(region, ())])
|
||||
if tok.value in places._COUNTRY_BY_NAME:
|
||||
more, extra = like_country(places._COUNTRY_BY_NAME[tok.value])
|
||||
sql, params = f"{sql} OR {more}", [*params, *extra]
|
||||
return f"({sql})", params
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def real() -> sqlite3.Connection:
|
||||
db = connect()
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, ?)",
|
||||
[(place, fold(place), country) for place, country in LINES])
|
||||
return db
|
||||
|
||||
|
||||
def test_there_are_enough_real_lines() -> None:
|
||||
assert len(set(LINES)) >= 300
|
||||
assert {"DK", "SE", "DE", "EU", ""} <= {country for _, country in LINES}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token", ORACLE_TOKENS)
|
||||
def test_matches_agrees_with_the_like_it_replaces(real: sqlite3.Connection, token: str) -> None:
|
||||
sql, params = like_sql(token)
|
||||
by_like = {row[0] for row in real.execute(
|
||||
f"SELECT rowid FROM roles WHERE {sql}", params)}
|
||||
by_function = {row[0] for row in real.execute(
|
||||
"SELECT rowid, place_fold, country FROM roles") if matches(row[1], row[2], token)}
|
||||
assert by_function == by_like, token
|
||||
sql, params = match_sql(token)
|
||||
assert {row[0] for row in real.execute(
|
||||
f"SELECT rowid FROM roles WHERE {sql}", params)} == by_like
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind, tokens", [
|
||||
("country", [t for t in ORACLE_TOKENS if t.startswith("country:")]),
|
||||
("region", [t for t in ORACLE_TOKENS if t.startswith("region:")]),
|
||||
("place", [t for t in ORACLE_TOKENS if t.startswith("place:")]),
|
||||
])
|
||||
def test_each_kind_finds_something_in_the_real_lines(real: sqlite3.Connection, kind: str,
|
||||
tokens: list[str]) -> None:
|
||||
"""Agreement on nothing would prove little: most tokens hit, some lines do not."""
|
||||
hits = [sum(matches(fold(place), country, t) for place, country in LINES) for t in tokens]
|
||||
assert sum(1 for n in hits if n) >= len(tokens) * 0.75, (kind, hits)
|
||||
assert all(n < len(LINES) for n in hits)
|
||||
|
||||
|
||||
def test_a_region_over_thousands_of_lines_is_fast() -> None:
|
||||
"""The reason `matches` is a function: 160 `LIKE` clauses took 400 ms per query.
|
||||
The bound is generous, a machine under load is not the test's business."""
|
||||
import time
|
||||
|
||||
db = connect()
|
||||
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
|
||||
rows = [(f"{place} {i}", fold(f"{place} {i}"), country)
|
||||
for i in range(10) for place, country in LINES]
|
||||
db.executemany("INSERT INTO roles VALUES (?, ?, ?)", rows)
|
||||
assert len(rows) >= 5000
|
||||
for token in ("region:oresund", "region:hovedstaden", "region:skane", "country:DK"):
|
||||
sql, params = match_sql(token)
|
||||
start = time.perf_counter()
|
||||
n = db.execute(f"SELECT count(*) FROM roles WHERE {sql}", params).fetchone()[0]
|
||||
assert time.perf_counter() - start < 0.5, token
|
||||
assert n > 0
|
||||
|
||||
|
||||
# --------------------------------------------- what is suggested, and on what
|
||||
|
||||
|
||||
def test_oresund_is_not_suggested_for_the_names_of_the_regions_it_holds() -> None:
|
||||
for q in ("ska", "skane", "den", "capital", "hoved", "scania"):
|
||||
assert "region:oresund" not in tokens(q), q
|
||||
for q in ("ore", "øre", "oresund", "öresund"):
|
||||
assert "region:oresund" in tokens(q), q
|
||||
|
||||
|
||||
def test_a_region_is_suggested_on_its_own_names() -> None:
|
||||
assert tokens("ska")[:1] == ["region:skane"]
|
||||
assert "region:hovedstaden" in tokens("den")
|
||||
assert "region:zealand" in tokens("den") or "region:zealand" in tokens("zea")
|
||||
|
||||
|
||||
def test_a_line_that_holds_a_town_counts_under_it() -> None:
|
||||
munich = [s for s in suggest("mun", PLACES) if s.kind == "place"]
|
||||
assert [s.token for s in munich] == ["place:munich"]
|
||||
# Munich 40, München 10, Office 5, (DE) 3, Munich - Berlin 2, Garching (Munich) 4.
|
||||
assert munich[0].count == 40 + 10 + 5 + 3 + 2 + 4
|
||||
assert munich[0].label == "Munich"
|
||||
|
||||
|
||||
def test_a_line_that_holds_two_towns_counts_under_both() -> None:
|
||||
berlin = [s for s in suggest("ber", PLACES) if s.kind == "place"]
|
||||
assert [(s.label, s.count) for s in berlin] == [("Berlin", 100 + 2)]
|
||||
|
||||
|
||||
def test_copenhagen_with_an_office_or_a_hub_is_copenhagen() -> None:
|
||||
towns = [s for s in suggest("cope", PLACES) if s.kind == "place"]
|
||||
assert [s.token for s in towns] == ["place:kobenhavn"]
|
||||
assert towns[0].count == 69 + 35 + 34 + 1 + 56 + 1 + 16 + 14 + 2
|
||||
for text in ("Copenhagen Office", "Hub", "MSO"):
|
||||
assert all(text not in s.label for s in towns)
|
||||
|
||||
|
||||
def test_a_town_the_tables_do_not_know_stays_as_it_is() -> None:
|
||||
labels = {s.label for q in "abcdefghijklmnopqrstuvwxyz"
|
||||
for s in suggest(q, PLACES, limit=999) if s.kind == "place"}
|
||||
assert {"Albertslund", "Billund", "Auckland"} <= labels
|
||||
assert not {"Copenhagen Office", "Munich Office", "Munich - Berlin", "Munich (DE)"} & labels
|
||||
|
||||
|
||||
def test_a_line_that_is_only_a_known_town_in_a_word_is_not_a_hit_for_another() -> None:
|
||||
"""`Lund` is not in `Billund`: a line holds a town as whole words or not at all."""
|
||||
assert [s.label for s in suggest("lund", PLACES) if s.kind == "place"] == ["Lund"]
|
||||
|
||||
|
||||
def test_a_bar_separates_places_like_a_semicolon() -> None:
|
||||
rows = [("Denmark | Remote", "DK", 1), ("Lund | Malmö", "SE", 2)]
|
||||
assert [s.label for s in suggest("den", rows) if s.kind == "place"] == []
|
||||
assert [(s.label, s.count) for s in suggest("mal", rows) if s.kind == "place"] \
|
||||
== [("Malmö", 2)]
|
||||
Reference in new issue
Block a user