Add places: folded whole-word place matching, country, region and town tokens, suggestions

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
WolfandClaude committed 2026-09-29 21:58:32 +02:00
1 parent 64f367cc18
commit ebb1f560de
2 files changed
+990

No files matched your search

+453
View File
@@ -0,0 +1,453 @@
"""`app.places`: folding, the three token kinds run as SQL, and the suggestions.
The rows are `place` strings as boards write them, taken from the live database.
Each token's SQL runs against an in-memory table shaped like `roles`."""
from __future__ import annotations
import sqlite3
import pytest
from app import places
from app.places import REGIONS, Suggestion, Token, fold, label, match_sql, parse, suggest
#: (place, country) as a board wrote them and `country_of` read them.
ROWS = [
("København", "DK"), ("Copenhagen, Denmark", "DK"), ("Humlebæk, Denmark", "DK"),
("København K", "DK"), ("København Ø", "DK"), ("Ballerup", "DK"), ("Denmark", "DK"),
("Aarhus N, Region Central Jutland, Denmark", "DK"), ("Aarhus C, Denmark", "DK"),
("Billund, Denmark", "DK"), ("Kgs. Lyngby", "DK"), ("Kongens Lyngby", "DK"),
("Måløv, Capital Region of Denmark, Denmark", "DK"), ("Søborg", "DK"),
("Kalundborg, Region Zealand, Denmark", "DK"), ("Albertslund", "DK"),
("Odense, REGION_DNK 1852 South Denmark, Denmark", "DK"), ("Nakskov", "DK"),
("Nordhavn og mulighed for hjemmearbejde", "DK"), ("Remote OR Copenhagen", "DK"),
("Malmö", "SE"), ("Lund", "SE"), ("Sweden - Lund", "SE"), ("Helsingborg", "SE"),
("Malmö, Skåne County, Sweden", "SE"), ("Skåne län, Sweden", "SE"),
("Höganäs, Sweden", "SE"), ("Stockholm, Sweden", "SE"), ("Gothenburg, Sweden", "SE"),
("Aarhus, Denmark; København, Denmark; Stockholm, Sweden", "EU"),
("Copenhagen; London; Warsaw; Milano", "EU"),
("Copenhagen; Warsaw; New York", ""),
("Berlin, Germany; Munich, Germany", "DE"), ("Berlin, Germany", "DE"),
("München", "DE"), ("Munich, Bavaria, Germany", "DE"), ("Köln", "DE"), ("Cologne", "DE"),
("Zürich", "CH"), ("Zurich, Switzerland", "CH"), ("Genève", "CH"), ("Geneva", "CH"),
("Wien", "AT"), ("Vienna, Austria", "AT"),
("Auckland, New Zealand", "NZ"), ("Remote", ""), ("Hybrid", ""), ("Europe", ""),
("Remote - Denmark", "DK"), ("", ""),
]
@pytest.fixture()
def table() -> sqlite3.Connection:
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
db.executemany("INSERT INTO roles VALUES (?, ?, ?)",
[(place, fold(place), country) for place, country in ROWS])
return db
def found(db: sqlite3.Connection, *tokens: str) -> set[str]:
"""The places the tokens find, joined with OR as a caller would."""
parts = [match_sql(token) for token in tokens]
sql = " OR ".join(sql for sql, _ in parts)
params = [p for _, params in parts for p in params]
return {row[0] for row in db.execute(f"SELECT place FROM roles WHERE {sql}", params)}
# ------------------------------------------------------------------- folding
@pytest.mark.parametrize("text, folded", [
("Skåne län", "skane lan"),
("København K", "kobenhavn k"),
("Kgs. Lyngby", "kgs lyngby"),
("Malmö", "malmo"),
("Åbyhøj", "abyhoj"),
("Ærø", "aero"),
("Düsseldorf", "dusseldorf"),
("Zürich", "zurich"),
("Wrocław", "wroclaw"),
("Höje-Taastrup, Denmark", "hoje taastrup denmark"),
(" Aarhus N ", "aarhus n"),
("", ""),
])
def test_fold(text: str, folded: str) -> None:
assert fold(text) == folded
def test_fold_turns_a_ring_and_umlauts_into_the_plain_letter() -> None:
assert fold("å") == "a"
assert (fold("ä"), fold("ö"), fold("ü")) == ("a", "o", "u")
# ------------------------------------------------------------------- parsing
def test_parse_reads_the_three_kinds() -> None:
assert parse("country:DK") == Token("country", "DK", "Denmark")
assert parse("region:skane") == Token("region", "skane", "Skåne")
assert parse("place:malmo") == Token("place", "malmo", "Malmö")
def test_parse_reads_bare_text_as_a_place() -> None:
assert parse("Skåne") == Token("place", "skane", "Skåne")
assert parse("Kgs. Lyngby").value == "kgs lyngby"
assert parse("Foo: Bar").kind == "place"
def test_parse_is_forgiving_about_case_and_names() -> None:
assert parse("Country:dk").value == "DK"
assert parse("country:Denmark").value == "DK"
assert parse("region:Skåne").value == "skane"
assert parse("region:Capital Region of Denmark").value == "hovedstaden"
assert parse("place:København").value == "kobenhavn"
def test_a_token_round_trips_through_its_string() -> None:
for text in ("country:DK", "region:oresund", "place:kgs lyngby"):
assert str(parse(text)) == text
def test_label_reads_a_chip_from_the_address() -> None:
assert label("country:SE") == "Sweden"
assert label("region:hovedstaden") == "Capital Region of Denmark"
assert label("place:kobenhavn") == "København"
assert label("place:copenhagen") == "Copenhagen"
assert label("place:malmo") == "Malmö"
assert label("place:tromso") == "Tromso"
assert label(parse("place:kgs lyngby")) == "Kgs. Lyngby"
assert label("country:ZZ") == "ZZ"
assert label("region:atlantis") == "atlantis"
# ---------------------------------------------------------------------- SQL
def test_a_place_is_matched_on_whole_words(table: sqlite3.Connection) -> None:
lund = found(table, "place:lund")
assert lund == {"Lund", "Sweden - Lund"}
assert "Billund, Denmark" not in lund and "Albertslund" not in lund
assert found(table, "place:billund") == {"Billund, Denmark"}
def test_a_multi_word_place_is_a_phrase(table: sqlite3.Connection) -> None:
assert found(table, "place:kgs lyngby") == {"Kgs. Lyngby", "Kongens Lyngby"}
assert found(table, "place:capital denmark") == set()
assert found(table, "place:denmark capital") == set()
def test_diacritics_and_spellings_meet(table: sqlite3.Connection) -> None:
assert found(table, "place:Skåne") == found(table, "place:skane")
assert "Malmö" in found(table, "place:malmo") and "Malmö" in found(table, "place:Malmö")
assert {"København", "København K"} <= found(table, "place:kobenhavn")
COPENHAGENS = {"København", "Copenhagen, Denmark", "København K", "Remote OR Copenhagen"}
@pytest.mark.parametrize("spelling", ["copenhagen", "kobenhavn", "kopenhamn", "kopenhagen",
"København", "Köpenhamn"])
def test_copenhagen_is_every_spelling(table: sqlite3.Connection, spelling: str) -> None:
assert found(table, f"place:{spelling}") >= COPENHAGENS
@pytest.mark.parametrize("group", places._ALIAS_GROUPS, ids=lambda g: g[0])
def test_every_alias_finds_every_other(group: tuple[str, ...]) -> None:
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
db.executemany("INSERT INTO roles VALUES (?, ?, '')",
[(name, fold(name)) for name in group])
for spelling in group:
sql, params = match_sql(f"place:{spelling}")
hit = {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)}
assert hit == set(group), spelling
def test_no_spelling_belongs_to_two_groups() -> None:
seen: dict[str, str] = {}
for group in places._ALIAS_GROUPS:
for name in group:
assert seen.setdefault(fold(name), group[0]) == group[0], name
@pytest.mark.parametrize("pair", [("Munich", "München"), ("Cologne", "Köln"),
("Vienna", "Wien"), ("Zurich", "Zürich"),
("Geneva", "Genève")])
def test_a_city_written_two_ways_is_found_either_way(
table: sqlite3.Connection, pair: tuple[str, str]) -> None:
one, other = pair
hit = found(table, f"place:{one}")
assert hit == found(table, f"place:{other}")
words = {w for place in hit for w in fold(place).split()}
assert {fold(one), fold(other)} <= words
def test_aa_and_the_ring_are_one_letter(table: sqlite3.Connection) -> None:
hit = found(table, "place:århus")
assert hit == found(table, "place:aarhus") == {
"Aarhus N, Region Central Jutland, Denmark", "Aarhus C, Denmark",
"Aarhus, Denmark; København, Denmark; Stockholm, Sweden"}
def test_a_line_with_several_places_is_found_by_each(table: sqlite3.Connection) -> None:
line = "Aarhus, Denmark; København, Denmark; Stockholm, Sweden"
for tok in ("place:aarhus", "place:kobenhavn", "place:stockholm", "place:copenhagen"):
assert line in found(table, tok), tok
def test_several_tokens_are_joined_by_or(table: sqlite3.Connection) -> None:
hit = found(table, "country:DK", "region:skane")
assert {"København", "Malmö", "Lund"} <= hit
assert "Stockholm, Sweden" not in hit
def test_a_country_is_its_code(table: sqlite3.Connection) -> None:
hit = found(table, "country:DK")
assert {"København", "Denmark", "Ballerup", "Nakskov", "Odense, REGION_DNK 1852 South Denmark, Denmark"} <= hit
assert "Malmö" not in hit and "Berlin, Germany" not in hit
assert "Auckland, New Zealand" not in hit
def test_a_country_is_also_a_line_that_names_it_when_country_is_unset(
table: sqlite3.Connection) -> None:
hit = found(table, "country:DK")
# `country` is EU: a board listing offices in several countries.
assert "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" in hit
# Only cities are named, and one of them is Danish.
assert "Copenhagen; London; Warsaw; Milano" in hit
assert "Copenhagen; Warsaw; New York" in hit
# The same words under a country that is set are that country's business.
assert found(table, "country:SE") >= {"Stockholm, Sweden", "Malmö"}
assert "Aarhus, Denmark; København, Denmark; Stockholm, Sweden" in found(table, "country:SE")
assert "Copenhagen, Denmark" in hit # its own country is DK
def test_a_country_name_does_not_reach_a_role_of_another_country() -> None:
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
db.executemany("INSERT INTO roles VALUES (?, ?, ?)", [
("Remote, Denmark", "remote denmark", "GB"), ("Copenhagen", "copenhagen", "DE"),
("Copenhagen", "copenhagen", "DK"), ("Nowhere, Denmark", "nowhere denmark", "")])
sql, params = match_sql("country:DK")
assert db.execute(f"SELECT country FROM roles WHERE {sql} ORDER BY country",
params).fetchall() == [("",), ("DK",)]
def test_country_eu_is_the_column_alone(table: sqlite3.Connection) -> None:
assert found(table, "country:EU") == {"Aarhus, Denmark; København, Denmark; Stockholm, Sweden",
"Copenhagen; London; Warsaw; Milano"}
def test_a_region_is_its_names_and_its_towns(table: sqlite3.Connection) -> None:
skane = found(table, "region:skane")
assert skane == {"Malmö", "Lund", "Sweden - Lund", "Helsingborg", "Höganäs, Sweden",
"Malmö, Skåne County, Sweden", "Skåne län, Sweden"}
assert found(table, "region:Skåne") == skane
hovedstaden = found(table, "region:hovedstaden")
assert {"København", "København K", "København Ø", "Copenhagen, Denmark", "Ballerup",
"Humlebæk, Denmark", "Kgs. Lyngby", "Kongens Lyngby", "Søborg", "Albertslund",
"Måløv, Capital Region of Denmark, Denmark", "Nordhavn og mulighed for hjemmearbejde",
} <= hovedstaden
assert not hovedstaden & {"Aarhus C, Denmark", "Billund, Denmark", "Malmö", "Nakskov"}
def test_zealand_is_the_danish_region_and_not_new_zealand(table: sqlite3.Connection) -> None:
assert found(table, "region:zealand") == {"Kalundborg, Region Zealand, Denmark", "Nakskov"}
def test_jutland_and_funen(table: sqlite3.Connection) -> None:
assert {"Billund, Denmark", "Aarhus C, Denmark"} <= found(table, "region:jutland")
assert found(table, "region:funen") == {"Odense, REGION_DNK 1852 South Denmark, Denmark"}
def test_oresund_is_the_two_capital_regions(table: sqlite3.Connection) -> None:
assert found(table, "region:oresund") == (
found(table, "region:hovedstaden") | found(table, "region:skane"))
def test_a_region_is_a_place_written_as_bare_text(table: sqlite3.Connection) -> None:
assert found(table, "Skåne") == found(table, "region:skane")
assert found(table, "place:scania") == found(table, "region:skane")
assert found(table, "Denmark") == found(table, "country:DK")
def test_a_token_that_names_nothing_finds_nothing(table: sqlite3.Connection) -> None:
assert found(table, "region:atlantis") == set()
assert found(table, "place:") == set()
assert found(table, "") == set()
assert found(table, "place:nowhere-in-particular") == set()
def test_the_columns_are_the_callers(table: sqlite3.Connection) -> None:
sql, params = match_sql("country:DK", place="r.pf", country="r.c")
assert "r.pf" in sql and "r.c " in sql and "place_fold" not in sql
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE r (pf TEXT, c TEXT)")
db.executemany("INSERT INTO r VALUES (?, ?)", [("lund", "SE"), ("billund", "DK")])
sql, params = match_sql("place:lund", place="r.pf", country="r.c")
assert db.execute(f"SELECT pf FROM r WHERE {sql}", params).fetchall() == [("lund",)]
def test_sql_is_parenthesised_and_has_one_parameter_per_question_mark() -> None:
for token in ("country:DK", "region:oresund", "place:copenhagen", "place:denmark",
"region:atlantis"):
sql, params = match_sql(token)
assert sql.startswith("(") and sql.endswith(")")
assert sql.count("?") == len(params), token
def test_a_wildcard_in_a_value_is_a_letter() -> None:
assert places._escape("50%_off\\") == "50\\%\\_off\\\\"
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE roles (place TEXT, place_fold TEXT, country TEXT)")
db.executemany("INSERT INTO roles VALUES (?, ?, '')", [("a b", "a b"), ("axb", "axb"),
("a_b", "a_b")])
for token in ("place:a_b", "place:a%b"):
sql, params = match_sql(token)
assert {r[0] for r in db.execute(f"SELECT place FROM roles WHERE {sql}", params)} \
== {"a b"}, token
# A column that is not folded still matches a literal underscore, not any character.
sql, params = places._words("place", ["a_b"])
assert db.execute(f"SELECT place FROM roles WHERE {sql}", params).fetchall() == [("a_b",)]
# ----------------------------------------------------------------- the tables
def test_every_region_has_names_and_towns() -> None:
assert set(REGIONS) == {"skane", "hovedstaden", "zealand", "oresund", "jutland", "funen"}
for key, region in REGIONS.items():
assert region.key == key and region.label and region.names and region.towns
def test_the_bare_word_zealand_is_no_name_since_new_zealand_has_it() -> None:
assert "zealand" not in places._PHRASES["zealand"]
def test_the_capital_and_scanian_towns_are_in_their_regions() -> None:
skane = {fold(t) for t in REGIONS["skane"].towns}
assert {"malmo", "lund", "helsingborg", "kristianstad", "landskrona", "trelleborg",
"angelholm", "hassleholm", "ystad", "eslov", "hoganas", "staffanstorp", "lomma",
"bjuv", "burlov", "kavlinge", "svedala", "vellinge"} <= skane
hoved = {fold(t) for t in REGIONS["hovedstaden"].towns}
assert {"kobenhavn", "kgs lyngby", "soborg", "ballerup", "malov", "bagsvaerd", "humlebaek",
"smorum", "nordhavn", "frederiksberg", "hillerod", "helsingor"} <= hoved
# ----------------------------------------------------------------- suggestions
PLACES = [
("København", "DK", 69), ("Copenhagen, Denmark", "DK", 56), ("København K", "DK", 35),
("København Ø", "DK", 34), ("Kgs. Lyngby", "DK", 16), ("Billund, Denmark", "DK", 18),
("Aarhus N, Region Central Jutland, Denmark", "DK", 26), ("Albertslund", "DK", 10),
("Denmark", "DK", 27), ("Remote OR Copenhagen", "DK", 1),
("Malmö", "SE", 214), ("Lund", "SE", 127), ("Sweden - Lund", "SE", 29),
("Malmö, Skåne County, Sweden", "SE", 3), ("Stockholm, Sweden", "SE", 287),
("Aarhus, Denmark; København, Denmark; Stockholm, Sweden", "EU", 1),
("Remote", "", 40), ("Hybrid", "", 12), ("Europe", "EU", 20), ("", "", 5),
("Berlin, Germany", "DE", 100), ("Auckland, New Zealand", "NZ", 25),
("The Hague, Netherlands", "NL", 9), ("Den Haag", "NL", 4),
]
def tokens(q: str, **kw) -> list[str]:
return [s.token for s in suggest(q, PLACES, **kw)]
def test_skane_is_suggested_however_it_is_typed() -> None:
for q in ("Skåne", "skane", "SKANE", "ska", "Scania", "scan"):
assert "region:skane" in tokens(q), q
hit = next(s for s in suggest("skane", PLACES) if s.token == "region:skane")
assert hit == Suggestion("region:skane", "Skåne", "region", 214 + 127 + 29 + 3)
def test_a_country_counts_the_roles_that_carry_its_code() -> None:
hit = suggest("den", PLACES)[0]
assert hit == Suggestion("country:DK", "Denmark", "country",
69 + 56 + 35 + 34 + 16 + 18 + 26 + 10 + 27 + 1)
assert tokens("swe")[0] == "country:SE"
assert tokens("sverige")[0] == "country:SE"
assert tokens("danmark")[0] == "country:DK"
def test_countries_come_first_then_regions_then_towns() -> None:
kinds = [s.kind for s in suggest("den", PLACES)]
assert kinds == sorted(kinds, key=["country", "region", "place"].index)
assert kinds[0] == "country" and "region" in kinds and kinds[-1] == "place"
def test_towns_are_grouped_and_labelled_by_their_commonest_spelling() -> None:
lund = [s for s in suggest("lund", PLACES) if s.kind == "place"]
assert lund == [Suggestion("place:lund", "Lund", "place", 127 + 29)]
kbh = [s for s in suggest("kob", PLACES) if s.kind == "place"]
assert kbh[0].label == "København" and kbh[0].token == "place:kobenhavn"
def test_a_postal_district_is_not_a_town_of_its_own() -> None:
labels = [s.label for s in suggest("kob", PLACES) if s.kind == "place"]
assert "København K" not in labels and "København Ø" not in labels
assert [s.label for s in suggest("aarhus", PLACES) if s.kind == "place"] == ["Aarhus"]
def test_spellings_of_one_town_are_one_suggestion() -> None:
towns = [s for s in suggest("cop", PLACES) if s.kind == "place"]
assert len(towns) == 1
# København, K and Ø, the one in a list of three towns; Copenhagen, and "Remote OR Copenhagen".
assert towns[0].label == "København" and towns[0].count == 69 + 35 + 34 + 1 + 56 + 1
assert [s.token for s in suggest("kobenhavn", PLACES) if s.kind == "place"] \
== [towns[0].token]
def test_only_the_first_comma_part_of_each_semicolon_part_is_a_town() -> None:
labels = {s.label for q in "abcdefghijklmnopqrstuvwxyz"
for s in suggest(q, PLACES, limit=999) if s.kind == "place"}
assert {"Aarhus", "Stockholm", "Malmö", "Lund", "Berlin"} <= labels
assert not {"Denmark", "Sweden", "Region Central Jutland", "Remote", "Hybrid", "Europe"} \
& labels
def test_remote_hybrid_and_country_names_are_no_town() -> None:
for q in ("rem", "hyb", "eur", "denm"):
assert [s for s in suggest(q, PLACES) if s.kind == "place"] == [], q
def test_a_word_prefix_finds_a_multi_word_town() -> None:
assert tokens("haag") == tokens("hague") == tokens("den haag") == tokens("the hag") \
== ["place:the hague"]
assert tokens("kgs ly") == ["place:kgs lyngby"]
assert tokens("lyngby") == ["place:kgs lyngby"]
def test_the_word_must_start_not_end() -> None:
assert "place:lund" not in tokens("und")
assert "place:billund" not in tokens("lund")
assert "place:albertslund" not in tokens("lund")
def test_new_zealand_is_a_country_and_zealand_a_region_only_when_it_has_roles() -> None:
assert tokens("zea")[:1] == ["country:NZ"]
assert "region:zealand" not in tokens("zea") # no Zealand role in PLACES
assert "country:AU" not in tokens("aus") # no role of its own
def test_limit_and_the_empty_query() -> None:
assert suggest("", PLACES) == [] and suggest(" ", PLACES) == [] and suggest("---", PLACES) == []
assert len(suggest("a", PLACES, limit=3)) == 3
assert len(suggest("a", PLACES)) <= 12
def test_towns_are_ordered_by_roles() -> None:
towns = [s for s in suggest("l", PLACES) if s.kind == "place"]
assert [s.count for s in towns] == sorted((s.count for s in towns), reverse=True)
def test_suggest_takes_any_iterable() -> None:
assert suggest("lund", iter(PLACES)) == suggest("lund", PLACES)
def test_a_suggested_token_finds_the_roles_it_counted(table: sqlite3.Connection) -> None:
for s in suggest("kob", PLACES) + suggest("lund", PLACES) + suggest("skane", PLACES):
assert parse(s.token).kind == s.kind
assert found(table, s.token), s.token
assert label(s.token) == s.label or s.kind == "place"