Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
122 lines
5.6 KiB
Python
122 lines
5.6 KiB
Python
"""The gazetteer's tables: seeding them from the code, reading them back, and
|
|
writing a role's places.
|
|
|
|
`places`, `place_aliases` and `place_includes` hold the places
|
|
(`app.places.seed` on a new or migrated file, and what a person adds later).
|
|
`role_places` holds one row per place of a role: the host's own text, its
|
|
fold, and the place it resolves to, NULL for a text the gazetteer does not
|
|
know. A text that names several places ("Malmö/Lund") is one row per place,
|
|
each with the same `position` and `text`. A row with a `url` is the place of
|
|
a copy folded into the role (`app/dupes.py`) and is not the role's own: only
|
|
own rows (`url = ''`) are rewritten here.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from collections.abc import Iterable, Mapping, Sequence
|
|
from typing import Any
|
|
|
|
from app import places
|
|
|
|
#: What changes when the gazetteer does: rows are only ever added.
|
|
VERSION = ("SELECT (SELECT MAX(rowid) FROM places), (SELECT MAX(rowid) FROM place_aliases), "
|
|
"(SELECT COUNT(*) FROM place_includes)")
|
|
|
|
|
|
def seed(conn: sqlite3.Connection) -> int:
|
|
"""Insert what `places.seed` describes, leaving rows already there alone.
|
|
Returns how many places were added."""
|
|
seeds, includes = places.seed()
|
|
ids = {(kind, key): place_id for place_id, kind, key in
|
|
conn.execute("SELECT id, kind, key FROM places")}
|
|
added = 0
|
|
for s in seeds:
|
|
if (s.kind, s.key) not in ids:
|
|
parent = ids[s.parent] if s.parent else None
|
|
ids[(s.kind, s.key)] = conn.execute(
|
|
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
|
|
(s.kind, s.key, s.name, parent)).lastrowid
|
|
added += 1
|
|
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
|
[(a, ids[(s.kind, s.key)]) for a in s.aliases])
|
|
conn.executemany("INSERT OR IGNORE INTO place_includes (place_id, part_id) VALUES (?, ?)",
|
|
[(ids[whole], ids[part]) for whole, part in includes])
|
|
return added
|
|
|
|
|
|
def load(conn: sqlite3.Connection) -> places.Gazetteer:
|
|
return places.Gazetteer(
|
|
[tuple(r) for r in conn.execute("SELECT id, kind, key, name, parent_id FROM places")],
|
|
[tuple(r) for r in conn.execute("SELECT fold, place_id FROM place_aliases")],
|
|
[tuple(r) for r in conn.execute("SELECT place_id, part_id FROM place_includes")])
|
|
|
|
|
|
def texts(raw: Mapping[str, Any]) -> list[str]:
|
|
"""A role's place texts from `Item.raw`: the `text` of each of `places`
|
|
(`[{"text": ..., "remote": ...}]`, the reader contract), else `place` split
|
|
at `;`, which is how the older readers join several places into one line.
|
|
Blank and repeated texts are dropped."""
|
|
given = raw.get("places")
|
|
if isinstance(given, list) and given:
|
|
found = [str((p.get("text") if isinstance(p, Mapping) else p) or "") for p in given]
|
|
else:
|
|
found = str(raw.get("place") or "").split(";")
|
|
return list(dict.fromkeys(t.strip() for t in found if t and t.strip()))
|
|
|
|
|
|
def rows(gaz: places.Gazetteer, uid: str, place_texts: Sequence[str]
|
|
) -> list[tuple[str, int, str, str, int | None]]:
|
|
"""`(uid, position, text, fold, place_id)` for each place of the texts."""
|
|
found: list[tuple[str, int, str, str, int | None]] = []
|
|
for position, text in enumerate(place_texts):
|
|
folded = places.fold(text)
|
|
found += [(uid, position, text, folded, place_id)
|
|
for place_id in (gaz.resolve(text) or (None,))]
|
|
return found
|
|
|
|
|
|
_INSERT = "INSERT INTO role_places (uid, position, text, fold, place_id) VALUES (?, ?, ?, ?, ?)"
|
|
|
|
|
|
def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
|
|
place_texts: Sequence[str]) -> None:
|
|
"""Replace one role's own places with these texts, resolved."""
|
|
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
|
|
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
|
|
|
|
|
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
|
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
|
"""`write` for many roles in one statement each way. Returns rows written."""
|
|
batch = list(by_uid)
|
|
conn.executemany("DELETE FROM role_places WHERE uid = ? AND url = ''",
|
|
[(uid,) for uid, _ in batch])
|
|
new = [row for uid, found in batch for row in rows(gaz, uid, found)]
|
|
conn.executemany(_INSERT, new)
|
|
return len(new)
|
|
|
|
|
|
def relink(conn: sqlite3.Connection, gaz: places.Gazetteer, words: str | None = None) -> int:
|
|
"""Resolve the own places of every role again, or only of the roles with a
|
|
place text that holds `words` (a fold) as whole words, after an alias or a
|
|
place was added. Returns how many roles now resolve differently."""
|
|
sql = "SELECT uid, position, text, place_id FROM role_places WHERE url = ''"
|
|
params: list[Any] = []
|
|
if words:
|
|
sql += " AND ' ' || fold || ' ' LIKE ? ESCAPE '\\'"
|
|
escaped = words.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
params.append(f"% {escaped} %")
|
|
sql = ("SELECT uid, position, text, place_id FROM role_places WHERE url = '' AND uid IN "
|
|
f"(SELECT uid FROM ({sql}))")
|
|
before: dict[str, dict[tuple[int, str], set[int | None]]] = {}
|
|
for uid, position, text, place_id in conn.execute(sql + " ORDER BY uid, position", params):
|
|
before.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
|
|
changed = []
|
|
for uid, held in before.items():
|
|
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in held}
|
|
if now != held:
|
|
changed.append((uid, [text for _, text in sorted(held)]))
|
|
write_many(conn, gaz, changed)
|
|
return len(changed)
|