Files
jobboard/tests/test_gazetteer.py
T

505 lines
22 KiB
Python

"""The gazetteer: its seed, the resolver, and a role's places in the store.
The store tests go through `Store`, as the feed and the gates do. The
real-line test compares the join over the gazetteer with the whole-word rule
it replaced (`places.matches`) on the live lines `test_places` holds: the
join must find every role the word rule found, but for the lines listed where
the word rule was wrong.
"""
from __future__ import annotations
import sqlite3
from collections import Counter
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
import pytest
from desk.models import Item
from app import places
from app.models import Gate, _CITIES, _COUNTRY_NAMES
from app.places import Gazetteer, fold, seed
from app.storage import Store
from test_places import LINES, ORACLE_TOKENS
from prepped import upsert
def gazetteer() -> Gazetteer:
seeds, includes = seed()
ids: dict[tuple[str, str], int] = {}
rows, aliases = [], []
for n, s in enumerate(seeds, 1):
ids[(s.kind, s.key)] = n
rows.append((n, s.kind, s.key, s.name, ids[s.parent] if s.parent else None))
aliases += [(a, n) for a in s.aliases]
return Gazetteer(rows, aliases, [(ids[a], ids[b]) for a, b in includes])
GAZ = gazetteer()
def names(text: str) -> list[str]:
return [GAZ.places[p].name for p in GAZ.resolve(text)]
# ---------------------------------------------------------------------- seed
def test_the_seed_holds_the_dictionaries_in_code() -> None:
seeds, includes = seed()
kinds = Counter(s.kind for s in seeds)
assert {s.key for s in seeds if s.kind == "continent"} == {"europe", "asia-pacific", "world"}
assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line)
keys = {(s.kind, s.key) for s in seeds}
assert {("region", key) for key in places.REGIONS} <= keys
assert {("region", "oresund"), ("region", "skane")} in [set(i) for i in includes]
folds = {a for s in seeds for a in s.aliases}
assert set(_CITIES) - {"region hovedstaden"} <= folds
assert {"kobenhavn", "kopenhamn", "copenhagen", "skane", "emea", "eu"} <= folds
def test_parents_come_before_their_children() -> None:
seen: set[tuple[str, str]] = set()
for s in seed()[0]:
assert s.parent is None or s.parent in seen, s
seen.add((s.kind, s.key))
def test_copenhagen_is_in_the_capital_region_in_denmark_in_europe() -> None:
copenhagen = GAZ.by_key[("city", "copenhagen")]
above = {GAZ.places[p].key for p in GAZ.ancestors(copenhagen)}
assert above == {"hovedstaden", "oresund", "DK", "europe"}
assert GAZ.country(copenhagen) == "DK"
assert GAZ.by_key[("country", "US")] not in GAZ.under({GAZ.by_key[("continent", "europe")]})
# ------------------------------------------------------------------ resolving
@pytest.mark.parametrize("text, expected", [
("Aarhus N, Region Central Jutland, Denmark", ["Aarhus"]),
("København K", ["Copenhagen"]),
("Copenhagen Office", ["Copenhagen"]),
("Malmö/Lund", ["Malmö", "Lund"]),
("Aalborg, Copenhagen, Kolding", ["Aalborg", "Copenhagen", "Kolding"]),
("Bopfingen, Germany", ["Germany"]),
("Skåne län, Sweden", ["Skåne"]),
("London, Ontario", ["Ontario"]),
("Bergen op Zoom, Netherlands", ["Netherlands"]),
("Mexicali, BAJA CALIFORNIA, Mexico", ["Mexico"]),
("Paris, TX", ["United States"]),
("Remote (DE)", ["Germany"]),
("London OR Paris OR Germany", ["London", "Paris", "Germany"]),
("Allerød or Stockholm (SE)", ["Allerød", "Stockholm"]),
("Finland, København, Litauen, Polen, Sverige", ["Finland", "Copenhagen", "Sweden"]),
("Remote, EU or US", ["Europe", "United States"]),
("Home based - EMEA", ["Europe"]),
("Luxembourg", ["Luxembourg"]),
("Nordhavn og mulighed for hjemmearbejde", ["Nordhavn"]),
("Remote", []),
("Oberkochen", []),
("", []),
])
def test_a_text_resolves_to_the_places_it_names(text: str, expected: list[str]) -> None:
assert names(text) == expected
def test_luxembourg_the_town_is_in_luxembourg_the_country() -> None:
town = GAZ.by_key[("city", "luxembourg")]
assert GAZ.resolve("Luxembourg") == (town,)
assert GAZ.country(town) == "LU"
@pytest.mark.parametrize("text", ["Remote", "Hybrid", "Multiple Locations", "2 Locations",
"Remote job", "", " "])
def test_a_text_of_how_a_role_is_worked_is_no_place(text: str) -> None:
assert places.placeless(text)
@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Lauchheim"])
def test_a_text_with_a_name_is_a_place(text: str) -> None:
assert not places.placeless(text)
# -------------------------------------------------------------------- tokens
def test_a_token_names_places_before_what_lies_in_them() -> None:
assert GAZ.lookup("country:DK") == {GAZ.by_key[("country", "DK")]}
assert GAZ.lookup("country:EU") == {GAZ.by_key[("continent", "europe")]}
assert GAZ.lookup("region:Skåne") == {GAZ.by_key[("region", "skane")]}
assert GAZ.lookup("place:kobenhavn") == {GAZ.by_key[("city", "copenhagen")]}
assert GAZ.lookup("place:denmark") == {GAZ.by_key[("country", "DK")]}
assert GAZ.lookup("place:oberkochen") is None
assert GAZ.lookup("country:ZZ") is None
def test_europe_is_every_european_country_and_eu() -> None:
under = GAZ.under(GAZ.lookup("country:EU") or ())
codes = GAZ.codes(under)
assert {"DK", "SE", "DE", "GB", "CH", "EU"} <= codes
assert not {"US", "CN"} & codes
def test_a_danish_region_as_akademikernes_jobbank_writes_it_is_placed() -> None:
# The labels of the site's search form. Unplaced, "Storkøbenhavn;
# Øresundsregionen" was known only as Øresund, which lies in no country.
assert names("Storkøbenhavn") == ["Capital Region of Denmark"]
assert names("Nordsjælland") == ["Capital Region of Denmark"]
for text in ("Østsjælland", "Vestsjælland", "Sydsjælland & Øer"):
assert names(text) == ["Zealand"]
for text in ("Sydøstjylland", "Sydvestjylland"):
assert names(text) == ["Jutland"]
assert names("Storkøbenhavn; Øresundsregionen") == ["Capital Region of Denmark", "Øresund"]
# "Abroad (other)" names no place, so it is not listed as unresolved.
assert names("Udlandet (øvrige)") == [] and places.placeless("Udlandet (øvrige)")
assert names("Udlandet (Sverige)") == ["Sweden"]
def test_oresund_holds_both_capital_regions() -> None:
under = GAZ.under(GAZ.lookup("region:oresund") or ())
assert {GAZ.by_key[("city", "malmo")], GAZ.by_key[("city", "copenhagen")]} <= under
assert GAZ.by_key[("city", "aarhus")] not in under
# --------------------------------------------------------------------- store
@pytest.fixture
def store(tmp_path: Path) -> Iterator[Store]:
yield Store(tmp_path / "roledesk.db")
def posting(source_id: str, *, title: str = "Backend Engineer", body: str = "",
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw=dict(raw))
item.source = "ats"
return item
def texts(store: Store, uid: str) -> list[tuple[int, str, str]]:
return [(p.position, p.text, p.name) for p in store.role_places(uid)]
def test_a_fresh_file_is_seeded(store: Store) -> None:
gaz = store.gazetteer()
assert len(gaz.places) == len(GAZ.places)
assert ("city", "copenhagen") in gaz.by_key
def test_a_joined_line_is_several_places(store: Store) -> None:
upsert(store, posting("1", place="Copenhagen; London; Warsaw"))
uid = posting("1").uid
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
(2, "Warsaw", "Warsaw")]
role = store.role(uid)
assert role is not None and role.place == "Copenhagen; London; Warsaw"
assert [p.name for p in role.places] == ["Copenhagen", "London", "Warsaw"]
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
upsert(store, posting("1", place="Berlin, Munich", places=[
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
{"text": "Remote (DE)", "remote": True}]))
assert texts(store, posting("1").uid) == [
(0, "Berlin", "Berlin"), (1, "Munich", "Munich"), (2, "Remote (DE)", "Germany")]
def test_places_without_a_line_make_the_line(store: Store) -> None:
upsert(store, posting("1", places=[{"text": "Lund", "remote": None},
{"text": "Malmö", "remote": None}]))
role = store.role(posting("1").uid)
assert role is not None
assert (role.place, role.country) == ("Lund; Malmö", "SE")
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
upsert(store, posting("1", place="Aarhus"))
upsert(store, posting("1"))
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
upsert(store, posting("1", place="Odense"))
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
upsert(store, posting("1", place="Aarhus"))
uid = posting("1").uid
with sqlite3.connect(store.path) as conn:
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
upsert(store, posting("1", place="Aalborg"))
assert [(p.text, p.url) for p in store.role_places(uid)] == [
("Aalborg", ""), ("Odense", "https://copy")]
def test_an_extracted_place_is_resolved(store: Store) -> None:
upsert(store, posting("1"))
uid = posting("1").uid
store.set_extracted(uid, {"place": "Lund, Sweden"})
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
# ---------------------------------------------------------- filter and gates
def uids(store: Store, *tokens: str) -> set[str]:
return {r.uid for r in store.roles(limit=100_000, places=list(tokens), by_score=False)}
@pytest.fixture
def several(store: Store) -> dict[str, str]:
lines = {"cph": "Copenhagen; London", "eu": "Berlin, Germany; Paris, France",
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
"zeiss": "Oberkochen; Lauchheim", "remote": "Remote (SE)", "ny": "New York, NY"}
for key, line in lines.items():
upsert(store, posting(key, place=line))
return {key: posting(key).uid for key in lines}
def test_a_country_is_every_place_in_it(store: Store, several: dict[str, str]) -> None:
assert uids(store, "country:DK") == {several["cph"], several["billund"]}
assert uids(store, "country:SE") == {several["malmo"], several["lund"], several["remote"]}
assert uids(store, "country:DE") == {several["eu"]}
assert uids(store, "country:GB") == {several["cph"]}
def test_europe_is_every_european_role(store: Store, several: dict[str, str]) -> None:
assert uids(store, "country:EU") == set(several.values()) - {several["ny"], several["zeiss"]}
def test_a_region_is_every_town_in_it(store: Store, several: dict[str, str]) -> None:
assert uids(store, "region:skane") == {several["malmo"], several["lund"]}
assert uids(store, "region:oresund") == {several["malmo"], several["lund"], several["cph"]}
assert uids(store, "place:skane") == uids(store, "region:skane")
def test_a_town_is_its_place_and_lund_is_not_in_billund(store: Store,
several: dict[str, str]) -> None:
assert uids(store, "place:lund") == {several["lund"]}
assert uids(store, "place:kobenhavn") == {several["cph"]}
def test_a_town_the_gazetteer_lacks_is_found_by_its_words(store: Store,
several: dict[str, str]) -> None:
assert uids(store, "place:oberkochen") == {several["zeiss"]}
def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
with pytest.raises(ValueError):
store.roles(places=["region:atlantis"])
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
upsert(store, posting("1", country="DK"))
assert uids(store, "country:DK") == {posting("1").uid}
def gates(store: Store, gate: Gate) -> dict[str, int]:
agent = store.save_agent("A", gates=[gate])
return {c.uid: c.gates[0] for c in store.score_candidates(agent.id, agent.gates)}
def test_a_country_gate_is_met_by_any_place_in_the_country(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="country", values=["DK", "SE"], mode="any"))
assert {uid for uid, g in found.items() if g == 1} == {
several[k] for k in ("cph", "billund", "malmo", "lund", "remote")}
# Only a role with neither a country nor a known place has no evidence.
assert {uid for uid, g in found.items() if g == -1} == {several["zeiss"]}
assert found[several["ny"]] == 0 and found[several["eu"]] == 0
def test_the_eu_value_of_a_country_gate_is_europe(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="country", values=["EU"]))
assert found[several["eu"]] == 1 and found[several["cph"]] == 1
assert found[several["ny"]] == 0
#: Lines of OpenAI and Mistral roles that agent 2's Europe gate was thought to
#: let through, with the country the board stated.
STATED = {"montreal": ("Montréal", "CA"), "sf": ("San Francisco", "US"),
"sf-ny": ("San Francisco; New York City", "US"),
"sf-london": ("San Francisco; London, UK", "US"),
"sf-seattle-london": ("San Francisco; Seattle; London, UK", "")}
def test_a_town_outside_europe_does_not_meet_a_europe_gate(store: Store) -> None:
"""Montréal is Canada's and San Francisco the US's, whatever the
stated country; a line that also names London is in Europe through it,
in the gate and in the Place filter alike."""
for key, (line, country) in STATED.items():
upsert(store, posting(key, place=line, country=country))
uid = {key: posting(key).uid for key in STATED}
found = gates(store, Gate(field="country", values=["DE", "FR", "GB", "IE", "EU"]))
met = {"sf-london", "sf-seattle-london"}
assert found == {uid[k]: int(k in met) for k in STATED}
assert uids(store, "country:EU") == {uid[k] for k in met}
assert uids(store, "country:CA") == {uid["montreal"]}
assert uids(store, "country:FR") == set()
def test_a_place_gate_matches_over_the_gazetteer(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="place", values=["Malmö", "Lund", "oberkochen"],
mode="any"))
assert {uid for uid, g in found.items() if g == 1} == {
several["malmo"], several["lund"], several["zeiss"]}
# ------------------------------------------------------------ unresolved texts
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
for n, line in enumerate(("Oberkochen; Lauchheim", "Oberkochen", "Remote", "Aalen; Berlin")):
upsert(store, posting(str(n), place=line))
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
("lauchheim", "Lauchheim", 1)]
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
def test_unlinked_texts_carry_their_roles_countries_and_companies(store: Store) -> None:
upsert(store, posting("1", place="Oberkochen", company="Zeiss", country="DE"))
upsert(store, posting("2", place="Oberkochen", company="Zeiss"))
upsert(store, posting("3", place="Oberkochen; Remote", company="Hensoldt", country="DE"))
upsert(store, posting("4", place="Remote"))
assert store.unlinked_places() == [{
"fold": "oberkochen", "text": "Oberkochen", "roles": 3,
"countries": [["DE", 2], ["", 1]], "companies": ["Zeiss", "Hensoldt"]}]
def test_an_alias_links_what_it_names(store: Store) -> None:
upsert(store, posting("1", place="Kbh. K"))
upsert(store, posting("2", place="Copenhagen"))
uid = posting("1").uid
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
assert texts(store, uid) == [(0, "Kbh. K", "Copenhagen")]
assert store.unresolved_places() == []
assert uids(store, "country:DK") == {uid, posting("2").uid}
# The second store, another process, sees the new alias too.
assert Store(store.path).gazetteer().lookup("place:kbh") == \
store.gazetteer().lookup("place:copenhagen")
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
upsert(store, posting("1", place="Oberkochen; Lauchheim"))
upsert(store, posting("2", place="Bopfingen, Germany"))
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
store.add_place("Bopfingen", "city", "country:DE")
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Lauchheim", "")]
assert texts(store, posting("2").uid) == [(0, "Bopfingen, Germany", "Bopfingen")]
assert uids(store, "place:oberkochen") == {posting("1").uid}
assert store.gazetteer().country(oberkochen) == "DE"
with pytest.raises(ValueError):
store.add_place("Oberkochen", "city", "country:DE")
with pytest.raises(ValueError):
store.add_place("Atlantis", "country", "country:EU")
def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
with pytest.raises(ValueError):
store.add_place_alias("Kbh", "place:atlantis")
with pytest.raises(ValueError):
store.add_place_alias("!!", "place:copenhagen")
def test_relink_places_resolves_every_role_again(store: Store) -> None:
upsert(store, posting("1", place="Oberkochen"))
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE role_places SET place_id = "
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
assert store.relink_places() == 1
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "")]
# --------------------------------------------------------- against the words
#: Lines of `test_places.LINES` the word rule found for a token and the
#: gazetteer does not, each rightly: the London in Canada, and a town named
#: after Zurich in Illinois.
NOT_FOUND_ANY_MORE: dict[str, set[str]] = {
"country:GB": {"London, Canada; Remote, United States; Oak Brook, United States"},
"place:london": {"London, Canada; Remote, United States; Oak Brook, United States"},
"place:zurich": {"Lake Zurich, Illinois, United States"},
}
@pytest.fixture(scope="module")
def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tuple[str, str]]]:
store = Store(tmp_path_factory.mktemp("real") / "roledesk.db")
lines = {}
for n, (place, country) in enumerate(LINES):
item = posting(f"line{n}", place=place, country=country)
upsert(store, item)
lines[item.uid] = (place, country)
return store, lines
@pytest.mark.parametrize("token", ORACLE_TOKENS)
def test_the_gazetteer_finds_what_the_words_found(
real: tuple[Store, dict[str, tuple[str, str]]], token: str) -> None:
store, lines = real
by_words = {uid for uid, (place, country) in lines.items()
if places.matches(fold(place), country, token)}
by_places = uids(store, token)
lost = {lines[uid][0] for uid in by_words - by_places}
assert lost <= NOT_FOUND_ANY_MORE.get(token, set()), (token, lost)
# ------------------------------------------------------------------ nearness
#: Agent 2's place gates: Europe as its scope, Denmark, Sweden, remote and
#: the Øresund towns as its `any` group.
EUROPE_DK_SE = [
Gate(field="country", values=["DK", "FR", "DE", "IE", "SE", "GB", "EU"], mode="require"),
Gate(field="country", values=["DK", "SE"], mode="any"),
Gate(field="remote", values=["remote"], mode="any"),
Gate(field="place", values=["Malmö", "copenhagen"], mode="any"),
]
def nearest(gates: list[Gate] | None, *texts: str) -> list[str]:
gaz = gazetteer()
near = places.Nearness(gaz, gates) if gates is not None else None
entries = [(t, list(gaz.resolve(t)) or [None]) for t in texts]
return [texts[i] for i in places.nearest_first(near, entries)]
def test_a_place_in_the_scope_leads_the_ones_outside_it() -> None:
"""Mistral's role in Montréal, Abu Dhabi, New York and Paris is a Paris
role for an agent kept to Europe; the others keep the host's order."""
assert nearest(EUROPE_DK_SE, "Montréal", "Abu Dhabi", "New York", "Paris") == [
"Paris", "Montréal", "Abu Dhabi", "New York"]
assert nearest(EUROPE_DK_SE, "San Francisco", "Dublin, Ireland") == [
"Dublin, Ireland", "San Francisco"]
def test_the_any_group_leads_the_scope_and_the_home_leads_the_group() -> None:
"""Copenhagen, Stockholm and Malmö meet the `any` group, London and
Lisbon only the scope; Denmark, the group's first value, is the home, so
a Danish place leads the group."""
assert nearest(EUROPE_DK_SE, "London", "Stockholm, Sweden", "Lisbon", "Copenhagen") == [
"Copenhagen", "Stockholm, Sweden", "London", "Lisbon"]
assert nearest(EUROPE_DK_SE, "London", "Malmö", "Copenhagen, Denmark") == [
"Copenhagen, Denmark", "Malmö", "London"]
def test_a_town_of_the_group_in_the_home_leads_the_rest_of_the_home() -> None:
"""Copenhagen is one of the group's towns and in Denmark, so it leads
"Denmark" and Billund, which lead Stockholm."""
assert nearest(EUROPE_DK_SE, "Stockholm", "Denmark", "Billund", "Copenhagen") == [
"Copenhagen", "Denmark", "Billund", "Stockholm"]
def test_a_remote_place_is_near_by_its_area() -> None:
"""A remote area within the scope meets the plain remote gate; one outside
it, and "Remote" with no area, come last."""
assert nearest(EUROPE_DK_SE, "Remote US", "Remote", "Berlin", "Remote - Europe") == [
"Remote - Europe", "Berlin", "Remote US", "Remote"]
def test_without_an_agent_or_its_gates_the_hosts_order_stands() -> None:
texts = ("Montréal", "New York", "Paris")
assert nearest(None, *texts) == list(texts)
assert nearest([], *texts) == list(texts)