Places as a gazetteer, role_places, teaser and three seniority sources
Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
bbd7522163
10 files changed
+1437
-98
No files matched your search
+156
-7
@@ -18,8 +18,9 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import country_of, posted_at, posted_host
|
||||
from app.models import SENIORITIES, country_of, posted_at, posted_host
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -44,8 +45,9 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- require the country: "UK | Remote" is not remote from Berlin.
|
||||
place TEXT NOT NULL DEFAULT '',
|
||||
place_key TEXT NOT NULL DEFAULT '',
|
||||
-- places.fold(place): what the Place filter and the place gate match
|
||||
-- whole words against, through the place_match function (app/places.py).
|
||||
-- places.fold(place): what a place: token the gazetteer does not know is
|
||||
-- matched against as whole words, through place_match (app/places.py).
|
||||
-- The filter and the gates otherwise read role_places.
|
||||
place_fold TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
@@ -53,7 +55,8 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
-- seniority.from_title(title) at ingest, until extract reads the posting.
|
||||
-- The level shown and filtered on: the first filled of seniority_model,
|
||||
-- seniority_stated and seniority_title (the last columns), written with them.
|
||||
seniority TEXT NOT NULL DEFAULT '',
|
||||
function TEXT NOT NULL DEFAULT '',
|
||||
-- Whether the posting says it sponsors a work permit, and the languages
|
||||
@@ -69,7 +72,7 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- stored the role; posted_at (app.models.posted_at) is the earlier of
|
||||
-- first_seen and published_at, trusted only for a host known to report a
|
||||
-- creation date rather than one that moves on an edit
|
||||
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
|
||||
-- (app.models.trusted_posted) — recency and the feed's newest order
|
||||
-- read posted_at, first_seen stays "when the app found it". closed_at is
|
||||
-- set when a board fetched in full no longer lists the role, and cleared
|
||||
-- if it returns.
|
||||
@@ -100,7 +103,62 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- precedes, which the migration tests do.
|
||||
dup_of TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
salary_text TEXT NOT NULL DEFAULT ''
|
||||
salary_text TEXT NOT NULL DEFAULT '',
|
||||
teaser TEXT NOT NULL DEFAULT '',
|
||||
seniority_model TEXT NOT NULL DEFAULT '',
|
||||
seniority_stated TEXT NOT NULL DEFAULT '',
|
||||
seniority_title TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
-- roles.teaser is the short text a list carries, kept until the full text
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
-- three levels behind roles.seniority: the model's (extract), the host's own
|
||||
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
|
||||
-- among the last columns: SQLite cannot drop a column a comment precedes,
|
||||
-- which the migration tests do.
|
||||
|
||||
-- The gazetteer: a place is a continent, a country (key the ISO code), a
|
||||
-- region or a city, in the place parent_id names. key is unique per kind:
|
||||
-- a region's key (app.places.REGIONS), a town's group. Seeded from the
|
||||
-- dictionaries in app/models.py and app/places.py (app.places.seed).
|
||||
CREATE TABLE IF NOT EXISTS places (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
|
||||
key TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
parent_id INTEGER REFERENCES places(id),
|
||||
UNIQUE (kind, key)
|
||||
);
|
||||
|
||||
-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn
|
||||
-- and copenhagen are Copenhagen. One fold may name several places
|
||||
-- (Luxembourg the town and the country).
|
||||
CREATE TABLE IF NOT EXISTS place_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
UNIQUE (fold, place_id)
|
||||
);
|
||||
|
||||
-- What a place holds beyond its children: Øresund holds the Capital Region
|
||||
-- of Denmark and Skåne, which lie in two countries.
|
||||
CREATE TABLE IF NOT EXISTS place_includes (
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (place_id, part_id)
|
||||
);
|
||||
|
||||
-- One row per place of a role (app/storage/gazetteer.py): the host's text in
|
||||
-- the host's order (position), its fold, and the place it resolves to, NULL
|
||||
-- while the gazetteer does not know it. A text naming several places is a
|
||||
-- row for each. url is the link of a copy folded into this role, '' for
|
||||
-- the role's own places.
|
||||
CREATE TABLE IF NOT EXISTS role_places (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER REFERENCES places(id),
|
||||
url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- The open vocabulary summarise reads off a posting: the role family, the
|
||||
@@ -303,6 +361,11 @@ CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
|
||||
-- Partial: all but about 3% of roles are no copy.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
ON roles(dup_of) WHERE dup_of IS NOT NULL;
|
||||
-- A role's places, and the roles in a set of places (the Place filter and
|
||||
-- the country and place gates, `uid IN (... WHERE place_id IN ...)`).
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -469,6 +532,10 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
# A Python step: the first rebuild runs the rule in app/dupes.py.
|
||||
(18, "roles.dup_of: one posting listed several times is one role, the others "
|
||||
"copies of it", lambda conn: _add_dup_of(conn)),
|
||||
# A Python step: the seed and the resolver are Python.
|
||||
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
|
||||
"places, resolved against a gazetteer; roles.teaser; the three seniority "
|
||||
"sources", lambda conn: _add_places(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -592,6 +659,81 @@ def _add_dup_of(conn: sqlite3.Connection) -> None:
|
||||
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
|
||||
|
||||
|
||||
_V19 = """
|
||||
ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT '';
|
||||
CREATE TABLE places (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
|
||||
key TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
parent_id INTEGER REFERENCES places(id),
|
||||
UNIQUE (kind, key)
|
||||
);
|
||||
CREATE TABLE place_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
UNIQUE (fold, place_id)
|
||||
);
|
||||
CREATE TABLE place_includes (
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (place_id, part_id)
|
||||
);
|
||||
CREATE TABLE role_places (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER REFERENCES places(id),
|
||||
url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def _add_places(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 19's step. The tables and columns, the gazetteer seeded, and
|
||||
every role's places: the `places` of its `raw` where a reader stated
|
||||
them, else its `place` column cut at `;` (not `raw`'s, which a pull that
|
||||
read the role's own page sends empty), each resolved once per distinct
|
||||
text.
|
||||
|
||||
The level splits into its sources without changing: on a role extract
|
||||
has read, `seniority` is the model's; on any other, it is the stated
|
||||
level when `raw` states one in the vocabulary, else the title's."""
|
||||
for statement in _statements(_V19):
|
||||
conn.execute(statement)
|
||||
gazetteer.seed(conn)
|
||||
gaz = gazetteer.load(conn)
|
||||
rows = conn.execute(
|
||||
"SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall()
|
||||
by_uid: list[tuple[str, list[str]]] = []
|
||||
levels = []
|
||||
for uid, raw, place, title, level, extracted_at in rows:
|
||||
try:
|
||||
given = json.loads(raw or "{}")
|
||||
except ValueError:
|
||||
given = {}
|
||||
given = given if isinstance(given, dict) else {}
|
||||
if found := gazetteer.texts({"places": given.get("places"), "place": place}):
|
||||
by_uid.append((uid, found))
|
||||
stated = str(given.get("seniority") or "")
|
||||
stated = stated if stated in SENIORITIES else ""
|
||||
if extracted_at is not None:
|
||||
levels.append((level, stated, from_title(title) if level else "", uid))
|
||||
elif stated:
|
||||
levels.append(("", stated, from_title(title), uid))
|
||||
elif level:
|
||||
levels.append(("", "", level, uid))
|
||||
written = gazetteer.write_many(conn, gaz, by_uid)
|
||||
conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, "
|
||||
"seniority_title = ? WHERE uid = ?", levels)
|
||||
log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d",
|
||||
len(gaz.places), written, len(by_uid), len(levels))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
@@ -659,6 +801,13 @@ def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def _create(conn: sqlite3.Connection) -> None:
|
||||
"""A new file: `SCHEMA`, and the gazetteer seeded."""
|
||||
for statement in _statements(SCHEMA):
|
||||
conn.execute(statement)
|
||||
gazetteer.seed(conn)
|
||||
|
||||
|
||||
def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
||||
fresh = conn.execute(
|
||||
@@ -668,7 +817,7 @@ def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
if fresh:
|
||||
# Atomic like a migration: a file half created would otherwise carry
|
||||
# the marker table and be taken for one to migrate.
|
||||
_apply(conn, LATEST, SCHEMA)
|
||||
_apply(conn, LATEST, _create)
|
||||
else:
|
||||
applied = migrate(conn)
|
||||
for statement in _statements(INDEXES):
|
||||
|
||||
Reference in new issue
Block a user