Places as a gazetteer, role_places, teaser and three seniority sources

Migration 19 adds places, place_aliases, place_includes (seeded from the
dictionaries in app/models.py and app/places.py, Europe as a continent over
the European countries) and role_places, one row per place of a role,
resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them
from raw["places"], else the place line cut at ";"; the migration builds them
from each role's place column.

The Place filter and the country and place gates are joins over the tree:
country:DK is any place under Denmark, region:skane any town in Skane, a
country gate is met by any place in its countries, and the EU fallback of
the word matcher is no longer used. A place: token the gazetteer does not
know still matches whole words. Store gains gazetteer, role_places,
unresolved_places, add_place_alias, add_place and relink_places.

roles.teaser is kept-filled and embedded while there is no description; a
vector is dropped when a better text arrives. seniority_model, _stated and
_title sit behind roles.seniority, which stays the first filled of them.

TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads
READS.fields.posted and falls back to the old set for undeclared readers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:31:45 +02:00
1 parent 4c0da58da7
commit bbd7522163
10 files changed
+1437 -98

No files matched your search

+156 -7
View File
@@ -18,8 +18,9 @@ import sqlite3
from collections.abc import Callable
from app import places
from app.models import country_of, posted_at, posted_host
from app.models import SENIORITIES, country_of, posted_at, posted_host
from app.seniority import from_title
from app.storage import gazetteer
log = logging.getLogger(__name__)
@@ -44,8 +45,9 @@ CREATE TABLE IF NOT EXISTS roles (
-- require the country: "UK | Remote" is not remote from Berlin.
place TEXT NOT NULL DEFAULT '',
place_key TEXT NOT NULL DEFAULT '',
-- places.fold(place): what the Place filter and the place gate match
-- whole words against, through the place_match function (app/places.py).
-- places.fold(place): what a place: token the gazetteer does not know is
-- matched against as whole words, through place_match (app/places.py).
-- The filter and the gates otherwise read role_places.
place_fold TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
@@ -53,7 +55,8 @@ CREATE TABLE IF NOT EXISTS roles (
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
-- seniority.from_title(title) at ingest, until extract reads the posting.
-- The level shown and filtered on: the first filled of seniority_model,
-- seniority_stated and seniority_title (the last columns), written with them.
seniority TEXT NOT NULL DEFAULT '',
function TEXT NOT NULL DEFAULT '',
-- Whether the posting says it sponsors a work permit, and the languages
@@ -69,7 +72,7 @@ CREATE TABLE IF NOT EXISTS roles (
-- stored the role; posted_at (app.models.posted_at) is the earlier of
-- first_seen and published_at, trusted only for a host known to report a
-- creation date rather than one that moves on an edit
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
-- (app.models.trusted_posted) — recency and the feed's newest order
-- read posted_at, first_seen stays "when the app found it". closed_at is
-- set when a board fetched in full no longer lists the role, and cleared
-- if it returns.
@@ -100,7 +103,62 @@ CREATE TABLE IF NOT EXISTS roles (
-- precedes, which the migration tests do.
dup_of TEXT,
created_at TEXT NOT NULL,
salary_text TEXT NOT NULL DEFAULT ''
salary_text TEXT NOT NULL DEFAULT '',
teaser TEXT NOT NULL DEFAULT '',
seniority_model TEXT NOT NULL DEFAULT '',
seniority_stated TEXT NOT NULL DEFAULT '',
seniority_title TEXT NOT NULL DEFAULT ''
);
-- roles.teaser is the short text a list carries, kept until the full text
-- arrives and embedded while there is none; a later empty teaser does not
-- wipe it. seniority_model, seniority_stated and seniority_title are the
-- three levels behind roles.seniority: the model's (extract), the host's own
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
-- among the last columns: SQLite cannot drop a column a comment precedes,
-- which the migration tests do.
-- The gazetteer: a place is a continent, a country (key the ISO code), a
-- region or a city, in the place parent_id names. key is unique per kind:
-- a region's key (app.places.REGIONS), a town's group. Seeded from the
-- dictionaries in app/models.py and app/places.py (app.places.seed).
CREATE TABLE IF NOT EXISTS places (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
key TEXT NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER REFERENCES places(id),
UNIQUE (kind, key)
);
-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn
-- and copenhagen are Copenhagen. One fold may name several places
-- (Luxembourg the town and the country).
CREATE TABLE IF NOT EXISTS place_aliases (
fold TEXT NOT NULL,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
UNIQUE (fold, place_id)
);
-- What a place holds beyond its children: Øresund holds the Capital Region
-- of Denmark and Skåne, which lie in two countries.
CREATE TABLE IF NOT EXISTS place_includes (
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (place_id, part_id)
);
-- One row per place of a role (app/storage/gazetteer.py): the host's text in
-- the host's order (position), its fold, and the place it resolves to, NULL
-- while the gazetteer does not know it. A text naming several places is a
-- row for each. url is the link of a copy folded into this role, '' for
-- the role's own places.
CREATE TABLE IF NOT EXISTS role_places (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
position INTEGER NOT NULL,
text TEXT NOT NULL,
fold TEXT NOT NULL,
place_id INTEGER REFERENCES places(id),
url TEXT NOT NULL DEFAULT ''
);
-- The open vocabulary summarise reads off a posting: the role family, the
@@ -303,6 +361,11 @@ CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
-- Partial: all but about 3% of roles are no copy.
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
ON roles(dup_of) WHERE dup_of IS NOT NULL;
-- A role's places, and the roles in a set of places (the Place filter and
-- the country and place gates, `uid IN (... WHERE place_id IN ...)`).
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -469,6 +532,10 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
# A Python step: the first rebuild runs the rule in app/dupes.py.
(18, "roles.dup_of: one posting listed several times is one role, the others "
"copies of it", lambda conn: _add_dup_of(conn)),
# A Python step: the seed and the resolver are Python.
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
"places, resolved against a gazetteer; roles.teaser; the three seniority "
"sources", lambda conn: _add_places(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -592,6 +659,81 @@ def _add_dup_of(conn: sqlite3.Connection) -> None:
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
_V19 = """
ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT '';
CREATE TABLE places (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
key TEXT NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER REFERENCES places(id),
UNIQUE (kind, key)
);
CREATE TABLE place_aliases (
fold TEXT NOT NULL,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
UNIQUE (fold, place_id)
);
CREATE TABLE place_includes (
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (place_id, part_id)
);
CREATE TABLE role_places (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
position INTEGER NOT NULL,
text TEXT NOT NULL,
fold TEXT NOT NULL,
place_id INTEGER REFERENCES places(id),
url TEXT NOT NULL DEFAULT ''
);
"""
def _add_places(conn: sqlite3.Connection) -> None:
"""Migration 19's step. The tables and columns, the gazetteer seeded, and
every role's places: the `places` of its `raw` where a reader stated
them, else its `place` column cut at `;` (not `raw`'s, which a pull that
read the role's own page sends empty), each resolved once per distinct
text.
The level splits into its sources without changing: on a role extract
has read, `seniority` is the model's; on any other, it is the stated
level when `raw` states one in the vocabulary, else the title's."""
for statement in _statements(_V19):
conn.execute(statement)
gazetteer.seed(conn)
gaz = gazetteer.load(conn)
rows = conn.execute(
"SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall()
by_uid: list[tuple[str, list[str]]] = []
levels = []
for uid, raw, place, title, level, extracted_at in rows:
try:
given = json.loads(raw or "{}")
except ValueError:
given = {}
given = given if isinstance(given, dict) else {}
if found := gazetteer.texts({"places": given.get("places"), "place": place}):
by_uid.append((uid, found))
stated = str(given.get("seniority") or "")
stated = stated if stated in SENIORITIES else ""
if extracted_at is not None:
levels.append((level, stated, from_title(title) if level else "", uid))
elif stated:
levels.append(("", stated, from_title(title), uid))
elif level:
levels.append(("", "", level, uid))
written = gazetteer.write_many(conn, gaz, by_uid)
conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, "
"seniority_title = ? WHERE uid = ?", levels)
log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d",
len(gaz.places), written, len(by_uid), len(levels))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
@@ -659,6 +801,13 @@ def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
conn.execute("COMMIT")
def _create(conn: sqlite3.Connection) -> None:
"""A new file: `SCHEMA`, and the gazetteer seeded."""
for statement in _statements(SCHEMA):
conn.execute(statement)
gazetteer.seed(conn)
def initialise(conn: sqlite3.Connection) -> list[int]:
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
fresh = conn.execute(
@@ -668,7 +817,7 @@ def initialise(conn: sqlite3.Connection) -> list[int]:
if fresh:
# Atomic like a migration: a file half created would otherwise carry
# the marker table and be taken for one to migrate.
_apply(conn, LATEST, SCHEMA)
_apply(conn, LATEST, _create)
else:
applied = migrate(conn)
for statement in _statements(INDEXES):