Files
jobboard/tests/test_gazetteer.py
T
WolfandClaude Opus 5.5 06ff9f549b List unlinked places and unmapped words under Sources, fix them in code with the unmapped skill
Settings > Sources > Unmapped shows the place texts no place resolves, with
their roles' countries and companies, and the host words no table maps, and
copies both as text. Prep fingerprints places.seed(), so a town or spelling
added to the code is seeded and every open role is mapped again at the next
prep. app prep unlinked lists the places on the command line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 13:06:22 +02:00

409 lines
17 KiB
Python

"""The gazetteer: its seed, the resolver, and a role's places in the store.
The store tests go through `Store`, as the feed and the gates do. The
real-line test compares the join over the gazetteer with the whole-word rule
it replaced (`places.matches`) on the live lines `test_places` holds: the
join must find every role the word rule found, but for the lines listed where
the word rule was wrong.
"""
from __future__ import annotations
import sqlite3
from collections import Counter
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
import pytest
from desk.models import Item
from app import places
from app.models import Gate, _CITIES, _COUNTRY_NAMES
from app.places import Gazetteer, fold, seed
from app.storage import Store
from test_places import LINES, ORACLE_TOKENS
from prepped import upsert
def gazetteer() -> Gazetteer:
seeds, includes = seed()
ids: dict[tuple[str, str], int] = {}
rows, aliases = [], []
for n, s in enumerate(seeds, 1):
ids[(s.kind, s.key)] = n
rows.append((n, s.kind, s.key, s.name, ids[s.parent] if s.parent else None))
aliases += [(a, n) for a in s.aliases]
return Gazetteer(rows, aliases, [(ids[a], ids[b]) for a, b in includes])
GAZ = gazetteer()
def names(text: str) -> list[str]:
return [GAZ.places[p].name for p in GAZ.resolve(text)]
# ---------------------------------------------------------------------- seed
def test_the_seed_holds_the_dictionaries_in_code() -> None:
seeds, includes = seed()
kinds = Counter(s.kind for s in seeds)
assert kinds["continent"] == 1
assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line)
keys = {(s.kind, s.key) for s in seeds}
assert {("region", key) for key in places.REGIONS} <= keys
assert {("region", "oresund"), ("region", "skane")} in [set(i) for i in includes]
folds = {a for s in seeds for a in s.aliases}
assert set(_CITIES) - {"region hovedstaden"} <= folds
assert {"kobenhavn", "kopenhamn", "copenhagen", "skane", "emea", "eu"} <= folds
def test_parents_come_before_their_children() -> None:
seen: set[tuple[str, str]] = set()
for s in seed()[0]:
assert s.parent is None or s.parent in seen, s
seen.add((s.kind, s.key))
def test_copenhagen_is_in_the_capital_region_in_denmark_in_europe() -> None:
copenhagen = GAZ.by_key[("city", "copenhagen")]
above = {GAZ.places[p].key for p in GAZ.ancestors(copenhagen)}
assert above == {"hovedstaden", "oresund", "DK", "europe"}
assert GAZ.country(copenhagen) == "DK"
assert GAZ.by_key[("country", "US")] not in GAZ.under({GAZ.by_key[("continent", "europe")]})
# ------------------------------------------------------------------ resolving
@pytest.mark.parametrize("text, expected", [
("Aarhus N, Region Central Jutland, Denmark", ["Aarhus"]),
("København K", ["Copenhagen"]),
("Copenhagen Office", ["Copenhagen"]),
("Malmö/Lund", ["Malmö", "Lund"]),
("Aalborg, Copenhagen, Kolding", ["Aalborg", "Copenhagen", "Kolding"]),
("Paderborn, Germany", ["Germany"]),
("Skåne län, Sweden", ["Skåne"]),
("London, Ontario", ["Ontario"]),
("Bergen op Zoom, Netherlands", ["Netherlands"]),
("Mexicali, BAJA CALIFORNIA, Mexico", ["Mexico"]),
("Paris, TX", ["United States"]),
("Remote (DE)", ["Germany"]),
("London OR Paris OR Germany", ["London", "Paris", "Germany"]),
("Allerød or Stockholm (SE)", ["Allerød", "Stockholm"]),
("Finland, København, Litauen, Polen, Sverige", ["Finland", "Copenhagen", "Sweden"]),
("Remote, EU or US", ["Europe", "United States"]),
("Home based - EMEA", ["Europe"]),
("Luxembourg", ["Luxembourg"]),
("Nordhavn og mulighed for hjemmearbejde", ["Nordhavn"]),
("Remote", []),
("Oberkochen", []),
("", []),
])
def test_a_text_resolves_to_the_places_it_names(text: str, expected: list[str]) -> None:
assert names(text) == expected
def test_luxembourg_the_town_is_in_luxembourg_the_country() -> None:
town = GAZ.by_key[("city", "luxembourg")]
assert GAZ.resolve("Luxembourg") == (town,)
assert GAZ.country(town) == "LU"
@pytest.mark.parametrize("text", ["Remote", "Hybrid", "Multiple Locations", "2 Locations",
"Remote job", "", " "])
def test_a_text_of_how_a_role_is_worked_is_no_place(text: str) -> None:
assert places.placeless(text)
@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Jena"])
def test_a_text_with_a_name_is_a_place(text: str) -> None:
assert not places.placeless(text)
# -------------------------------------------------------------------- tokens
def test_a_token_names_places_before_what_lies_in_them() -> None:
assert GAZ.lookup("country:DK") == {GAZ.by_key[("country", "DK")]}
assert GAZ.lookup("country:EU") == {GAZ.by_key[("continent", "europe")]}
assert GAZ.lookup("region:Skåne") == {GAZ.by_key[("region", "skane")]}
assert GAZ.lookup("place:kobenhavn") == {GAZ.by_key[("city", "copenhagen")]}
assert GAZ.lookup("place:denmark") == {GAZ.by_key[("country", "DK")]}
assert GAZ.lookup("place:oberkochen") is None
assert GAZ.lookup("country:ZZ") is None
def test_europe_is_every_european_country_and_eu() -> None:
under = GAZ.under(GAZ.lookup("country:EU") or ())
codes = GAZ.codes(under)
assert {"DK", "SE", "DE", "GB", "CH", "EU"} <= codes
assert not {"US", "CN"} & codes
def test_oresund_holds_both_capital_regions() -> None:
under = GAZ.under(GAZ.lookup("region:oresund") or ())
assert {GAZ.by_key[("city", "malmo")], GAZ.by_key[("city", "copenhagen")]} <= under
assert GAZ.by_key[("city", "aarhus")] not in under
# --------------------------------------------------------------------- store
@pytest.fixture
def store(tmp_path: Path) -> Iterator[Store]:
yield Store(tmp_path / "roledesk.db")
def posting(source_id: str, *, title: str = "Backend Engineer", body: str = "",
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw=dict(raw))
item.source = "ats"
return item
def texts(store: Store, uid: str) -> list[tuple[int, str, str]]:
return [(p.position, p.text, p.name) for p in store.role_places(uid)]
def test_a_fresh_file_is_seeded(store: Store) -> None:
gaz = store.gazetteer()
assert len(gaz.places) == len(GAZ.places)
assert ("city", "copenhagen") in gaz.by_key
def test_a_joined_line_is_several_places(store: Store) -> None:
upsert(store, posting("1", place="Copenhagen; London; Warsaw"))
uid = posting("1").uid
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
(2, "Warsaw", "Warsaw")]
role = store.role(uid)
assert role is not None and role.place == "Copenhagen; London; Warsaw"
assert [p.name for p in role.places] == ["Copenhagen", "London", "Warsaw"]
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
upsert(store, posting("1", place="Berlin, Munich", places=[
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
{"text": "Remote (DE)", "remote": True}]))
assert texts(store, posting("1").uid) == [
(0, "Berlin", "Berlin"), (1, "Munich", "Munich"), (2, "Remote (DE)", "Germany")]
def test_places_without_a_line_make_the_line(store: Store) -> None:
upsert(store, posting("1", places=[{"text": "Lund", "remote": None},
{"text": "Malmö", "remote": None}]))
role = store.role(posting("1").uid)
assert role is not None
assert (role.place, role.country) == ("Lund; Malmö", "SE")
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
upsert(store, posting("1", place="Aarhus"))
upsert(store, posting("1"))
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
upsert(store, posting("1", place="Odense"))
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
upsert(store, posting("1", place="Aarhus"))
uid = posting("1").uid
with sqlite3.connect(store.path) as conn:
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
upsert(store, posting("1", place="Aalborg"))
assert [(p.text, p.url) for p in store.role_places(uid)] == [
("Aalborg", ""), ("Odense", "https://copy")]
def test_an_extracted_place_is_resolved(store: Store) -> None:
upsert(store, posting("1"))
uid = posting("1").uid
store.set_extracted(uid, {"place": "Lund, Sweden"})
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
# ---------------------------------------------------------- filter and gates
def uids(store: Store, *tokens: str) -> set[str]:
return {r.uid for r in store.roles(limit=100_000, places=list(tokens), by_score=False)}
@pytest.fixture
def several(store: Store) -> dict[str, str]:
lines = {"cph": "Copenhagen; London", "eu": "Berlin, Germany; Paris, France",
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
for key, line in lines.items():
upsert(store, posting(key, place=line))
return {key: posting(key).uid for key in lines}
def test_a_country_is_every_place_in_it(store: Store, several: dict[str, str]) -> None:
assert uids(store, "country:DK") == {several["cph"], several["billund"]}
assert uids(store, "country:SE") == {several["malmo"], several["lund"], several["remote"]}
assert uids(store, "country:DE") == {several["eu"]}
assert uids(store, "country:GB") == {several["cph"]}
def test_europe_is_every_european_role(store: Store, several: dict[str, str]) -> None:
assert uids(store, "country:EU") == set(several.values()) - {several["ny"], several["zeiss"]}
def test_a_region_is_every_town_in_it(store: Store, several: dict[str, str]) -> None:
assert uids(store, "region:skane") == {several["malmo"], several["lund"]}
assert uids(store, "region:oresund") == {several["malmo"], several["lund"], several["cph"]}
assert uids(store, "place:skane") == uids(store, "region:skane")
def test_a_town_is_its_place_and_lund_is_not_in_billund(store: Store,
several: dict[str, str]) -> None:
assert uids(store, "place:lund") == {several["lund"]}
assert uids(store, "place:kobenhavn") == {several["cph"]}
def test_a_town_the_gazetteer_lacks_is_found_by_its_words(store: Store,
several: dict[str, str]) -> None:
assert uids(store, "place:oberkochen") == {several["zeiss"]}
def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
with pytest.raises(ValueError):
store.roles(places=["region:atlantis"])
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
upsert(store, posting("1", country="DK"))
assert uids(store, "country:DK") == {posting("1").uid}
def gates(store: Store, gate: Gate) -> dict[str, int]:
agent = store.save_agent("A", gates=[gate])
return {c.uid: c.gates[0] for c in store.score_candidates(agent.id, agent.gates)}
def test_a_country_gate_is_met_by_any_place_in_the_country(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="country", values=["DK", "SE"], mode="any"))
assert {uid for uid, g in found.items() if g == 1} == {
several[k] for k in ("cph", "billund", "malmo", "lund", "remote")}
# Only a role with neither a country nor a known place has no evidence.
assert {uid for uid, g in found.items() if g == -1} == {several["zeiss"]}
assert found[several["ny"]] == 0 and found[several["eu"]] == 0
def test_the_eu_value_of_a_country_gate_is_europe(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="country", values=["EU"]))
assert found[several["eu"]] == 1 and found[several["cph"]] == 1
assert found[several["ny"]] == 0
def test_a_place_gate_matches_over_the_gazetteer(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="place", values=["Malmö", "Lund", "oberkochen"],
mode="any"))
assert {uid for uid, g in found.items() if g == 1} == {
several["malmo"], several["lund"], several["zeiss"]}
# ------------------------------------------------------------ unresolved texts
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
upsert(store, posting(str(n), place=line))
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
("jena", "Jena", 1)]
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
def test_unlinked_texts_carry_their_roles_countries_and_companies(store: Store) -> None:
upsert(store, posting("1", place="Oberkochen", company="Zeiss", country="DE"))
upsert(store, posting("2", place="Oberkochen", company="Zeiss"))
upsert(store, posting("3", place="Oberkochen; Remote", company="Hensoldt", country="DE"))
upsert(store, posting("4", place="Remote"))
assert store.unlinked_places() == [{
"fold": "oberkochen", "text": "Oberkochen", "roles": 3,
"countries": [["DE", 2], ["", 1]], "companies": ["Zeiss", "Hensoldt"]}]
def test_an_alias_links_what_it_names(store: Store) -> None:
upsert(store, posting("1", place="Kbh. K"))
upsert(store, posting("2", place="Copenhagen"))
uid = posting("1").uid
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
assert texts(store, uid) == [(0, "Kbh. K", "Copenhagen")]
assert store.unresolved_places() == []
assert uids(store, "country:DK") == {uid, posting("2").uid}
# The second store, another process, sees the new alias too.
assert Store(store.path).gazetteer().lookup("place:kbh") == \
store.gazetteer().lookup("place:copenhagen")
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
upsert(store, posting("1", place="Oberkochen; Jena"))
upsert(store, posting("2", place="Paderborn, Germany"))
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
store.add_place("Paderborn", "city", "country:DE")
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
assert texts(store, posting("2").uid) == [(0, "Paderborn, Germany", "Paderborn")]
assert uids(store, "place:oberkochen") == {posting("1").uid}
assert store.gazetteer().country(oberkochen) == "DE"
with pytest.raises(ValueError):
store.add_place("Oberkochen", "city", "country:DE")
with pytest.raises(ValueError):
store.add_place("Atlantis", "country", "country:EU")
def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
with pytest.raises(ValueError):
store.add_place_alias("Kbh", "place:atlantis")
with pytest.raises(ValueError):
store.add_place_alias("!!", "place:copenhagen")
def test_relink_places_resolves_every_role_again(store: Store) -> None:
upsert(store, posting("1", place="Oberkochen"))
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE role_places SET place_id = "
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
assert store.relink_places() == 1
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "")]
# --------------------------------------------------------- against the words
#: Lines of `test_places.LINES` the word rule found for a token and the
#: gazetteer does not, each rightly: the London in Canada, and a town named
#: after Zurich in Illinois.
NOT_FOUND_ANY_MORE: dict[str, set[str]] = {
"country:GB": {"London, Canada; Remote, United States; Oak Brook, United States"},
"place:london": {"London, Canada; Remote, United States; Oak Brook, United States"},
"place:zurich": {"Lake Zurich, Illinois, United States"},
}
@pytest.fixture(scope="module")
def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tuple[str, str]]]:
store = Store(tmp_path_factory.mktemp("real") / "roledesk.db")
lines = {}
for n, (place, country) in enumerate(LINES):
item = posting(f"line{n}", place=place, country=country)
upsert(store, item)
lines[item.uid] = (place, country)
return store, lines
@pytest.mark.parametrize("token", ORACLE_TOKENS)
def test_the_gazetteer_finds_what_the_words_found(
real: tuple[Store, dict[str, tuple[str, str]]], token: str) -> None:
store, lines = real
by_words = {uid for uid, (place, country) in lines.items()
if places.matches(fold(place), country, token)}
by_places = uids(store, token)
lost = {lines[uid][0] for uid in by_words - by_places}
assert lost <= NOT_FOUND_ANY_MORE.get(token, set()), (token, lost)