Places as a gazetteer, role_places, teaser and three seniority sources

Migration 19 adds places, place_aliases, place_includes (seeded from the
dictionaries in app/models.py and app/places.py, Europe as a continent over
the European countries) and role_places, one row per place of a role,
resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them
from raw["places"], else the place line cut at ";"; the migration builds them
from each role's place column.

The Place filter and the country and place gates are joins over the tree:
country:DK is any place under Denmark, region:skane any town in Skane, a
country gate is met by any place in its countries, and the EU fallback of
the word matcher is no longer used. A place: token the gazetteer does not
know still matches whole words. Store gains gazetteer, role_places,
unresolved_places, add_place_alias, add_place and relink_places.

roles.teaser is kept-filled and embedded while there is no description; a
vector is dropped when a better text arrives. seniority_model, _stated and
_title sit behind roles.seniority, which stays the first filled of them.

TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads
READS.fields.posted and falls back to the old set for undeclared readers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:31:45 +02:00
1 parent 4c0da58da7
commit bbd7522163
10 files changed
+1437 -98

No files matched your search

+2
View File
@@ -3,6 +3,8 @@
schema.py the tables, and the numbered migrations that bring an older
file to the current shape
store.py `Store`, every SQL statement in the application
gazetteer.py the places tables: seeded from the code, read back, and a
role's places written and relinked
Written for roledesk rather than imported from newsdesk: `desk` has no storage
package on purpose, and the connection plumbing here is a deliberate copy.
+121
View File
@@ -0,0 +1,121 @@
"""The gazetteer's tables: seeding them from the code, reading them back, and
writing a role's places.
`places`, `place_aliases` and `place_includes` hold the places
(`app.places.seed` on a new or migrated file, and what a person adds later).
`role_places` holds one row per place of a role: the host's own text, its
fold, and the place it resolves to, NULL for a text the gazetteer does not
know. A text that names several places ("Malmö/Lund") is one row per place,
each with the same `position` and `text`. A row with a `url` is the place of
a copy folded into the role (`app/dupes.py`) and is not the role's own: only
own rows (`url = ''`) are rewritten here.
"""
from __future__ import annotations
import sqlite3
from collections.abc import Iterable, Mapping, Sequence
from typing import Any
from app import places
#: What changes when the gazetteer does: rows are only ever added.
VERSION = ("SELECT (SELECT MAX(rowid) FROM places), (SELECT MAX(rowid) FROM place_aliases), "
"(SELECT COUNT(*) FROM place_includes)")
def seed(conn: sqlite3.Connection) -> int:
"""Insert what `places.seed` describes, leaving rows already there alone.
Returns how many places were added."""
seeds, includes = places.seed()
ids = {(kind, key): place_id for place_id, kind, key in
conn.execute("SELECT id, kind, key FROM places")}
added = 0
for s in seeds:
if (s.kind, s.key) not in ids:
parent = ids[s.parent] if s.parent else None
ids[(s.kind, s.key)] = conn.execute(
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
(s.kind, s.key, s.name, parent)).lastrowid
added += 1
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
[(a, ids[(s.kind, s.key)]) for a in s.aliases])
conn.executemany("INSERT OR IGNORE INTO place_includes (place_id, part_id) VALUES (?, ?)",
[(ids[whole], ids[part]) for whole, part in includes])
return added
def load(conn: sqlite3.Connection) -> places.Gazetteer:
return places.Gazetteer(
[tuple(r) for r in conn.execute("SELECT id, kind, key, name, parent_id FROM places")],
[tuple(r) for r in conn.execute("SELECT fold, place_id FROM place_aliases")],
[tuple(r) for r in conn.execute("SELECT place_id, part_id FROM place_includes")])
def texts(raw: Mapping[str, Any]) -> list[str]:
"""A role's place texts from `Item.raw`: the `text` of each of `places`
(`[{"text": ..., "remote": ...}]`, the reader contract), else `place` split
at `;`, which is how the older readers join several places into one line.
Blank and repeated texts are dropped."""
given = raw.get("places")
if isinstance(given, list) and given:
found = [str((p.get("text") if isinstance(p, Mapping) else p) or "") for p in given]
else:
found = str(raw.get("place") or "").split(";")
return list(dict.fromkeys(t.strip() for t in found if t and t.strip()))
def rows(gaz: places.Gazetteer, uid: str, place_texts: Sequence[str]
) -> list[tuple[str, int, str, str, int | None]]:
"""`(uid, position, text, fold, place_id)` for each place of the texts."""
found: list[tuple[str, int, str, str, int | None]] = []
for position, text in enumerate(place_texts):
folded = places.fold(text)
found += [(uid, position, text, folded, place_id)
for place_id in (gaz.resolve(text) or (None,))]
return found
_INSERT = "INSERT INTO role_places (uid, position, text, fold, place_id) VALUES (?, ?, ?, ?, ?)"
def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
place_texts: Sequence[str]) -> None:
"""Replace one role's own places with these texts, resolved."""
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
"""`write` for many roles in one statement each way. Returns rows written."""
batch = list(by_uid)
conn.executemany("DELETE FROM role_places WHERE uid = ? AND url = ''",
[(uid,) for uid, _ in batch])
new = [row for uid, found in batch for row in rows(gaz, uid, found)]
conn.executemany(_INSERT, new)
return len(new)
def relink(conn: sqlite3.Connection, gaz: places.Gazetteer, words: str | None = None) -> int:
"""Resolve the own places of every role again, or only of the roles with a
place text that holds `words` (a fold) as whole words, after an alias or a
place was added. Returns how many roles now resolve differently."""
sql = "SELECT uid, position, text, place_id FROM role_places WHERE url = ''"
params: list[Any] = []
if words:
sql += " AND ' ' || fold || ' ' LIKE ? ESCAPE '\\'"
escaped = words.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
params.append(f"% {escaped} %")
sql = ("SELECT uid, position, text, place_id FROM role_places WHERE url = '' AND uid IN "
f"(SELECT uid FROM ({sql}))")
before: dict[str, dict[tuple[int, str], set[int | None]]] = {}
for uid, position, text, place_id in conn.execute(sql + " ORDER BY uid, position", params):
before.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
changed = []
for uid, held in before.items():
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in held}
if now != held:
changed.append((uid, [text for _, text in sorted(held)]))
write_many(conn, gaz, changed)
return len(changed)
+156 -7
View File
@@ -18,8 +18,9 @@ import sqlite3
from collections.abc import Callable
from app import places
from app.models import country_of, posted_at, posted_host
from app.models import SENIORITIES, country_of, posted_at, posted_host
from app.seniority import from_title
from app.storage import gazetteer
log = logging.getLogger(__name__)
@@ -44,8 +45,9 @@ CREATE TABLE IF NOT EXISTS roles (
-- require the country: "UK | Remote" is not remote from Berlin.
place TEXT NOT NULL DEFAULT '',
place_key TEXT NOT NULL DEFAULT '',
-- places.fold(place): what the Place filter and the place gate match
-- whole words against, through the place_match function (app/places.py).
-- places.fold(place): what a place: token the gazetteer does not know is
-- matched against as whole words, through place_match (app/places.py).
-- The filter and the gates otherwise read role_places.
place_fold TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
@@ -53,7 +55,8 @@ CREATE TABLE IF NOT EXISTS roles (
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
-- seniority.from_title(title) at ingest, until extract reads the posting.
-- The level shown and filtered on: the first filled of seniority_model,
-- seniority_stated and seniority_title (the last columns), written with them.
seniority TEXT NOT NULL DEFAULT '',
function TEXT NOT NULL DEFAULT '',
-- Whether the posting says it sponsors a work permit, and the languages
@@ -69,7 +72,7 @@ CREATE TABLE IF NOT EXISTS roles (
-- stored the role; posted_at (app.models.posted_at) is the earlier of
-- first_seen and published_at, trusted only for a host known to report a
-- creation date rather than one that moves on an edit
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
-- (app.models.trusted_posted) — recency and the feed's newest order
-- read posted_at, first_seen stays "when the app found it". closed_at is
-- set when a board fetched in full no longer lists the role, and cleared
-- if it returns.
@@ -100,7 +103,62 @@ CREATE TABLE IF NOT EXISTS roles (
-- precedes, which the migration tests do.
dup_of TEXT,
created_at TEXT NOT NULL,
salary_text TEXT NOT NULL DEFAULT ''
salary_text TEXT NOT NULL DEFAULT '',
teaser TEXT NOT NULL DEFAULT '',
seniority_model TEXT NOT NULL DEFAULT '',
seniority_stated TEXT NOT NULL DEFAULT '',
seniority_title TEXT NOT NULL DEFAULT ''
);
-- roles.teaser is the short text a list carries, kept until the full text
-- arrives and embedded while there is none; a later empty teaser does not
-- wipe it. seniority_model, seniority_stated and seniority_title are the
-- three levels behind roles.seniority: the model's (extract), the host's own
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
-- among the last columns: SQLite cannot drop a column a comment precedes,
-- which the migration tests do.
-- The gazetteer: a place is a continent, a country (key the ISO code), a
-- region or a city, in the place parent_id names. key is unique per kind:
-- a region's key (app.places.REGIONS), a town's group. Seeded from the
-- dictionaries in app/models.py and app/places.py (app.places.seed).
CREATE TABLE IF NOT EXISTS places (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
key TEXT NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER REFERENCES places(id),
UNIQUE (kind, key)
);
-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn
-- and copenhagen are Copenhagen. One fold may name several places
-- (Luxembourg the town and the country).
CREATE TABLE IF NOT EXISTS place_aliases (
fold TEXT NOT NULL,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
UNIQUE (fold, place_id)
);
-- What a place holds beyond its children: Øresund holds the Capital Region
-- of Denmark and Skåne, which lie in two countries.
CREATE TABLE IF NOT EXISTS place_includes (
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (place_id, part_id)
);
-- One row per place of a role (app/storage/gazetteer.py): the host's text in
-- the host's order (position), its fold, and the place it resolves to, NULL
-- while the gazetteer does not know it. A text naming several places is a
-- row for each. url is the link of a copy folded into this role, '' for
-- the role's own places.
CREATE TABLE IF NOT EXISTS role_places (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
position INTEGER NOT NULL,
text TEXT NOT NULL,
fold TEXT NOT NULL,
place_id INTEGER REFERENCES places(id),
url TEXT NOT NULL DEFAULT ''
);
-- The open vocabulary summarise reads off a posting: the role family, the
@@ -303,6 +361,11 @@ CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
-- Partial: all but about 3% of roles are no copy.
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
ON roles(dup_of) WHERE dup_of IS NOT NULL;
-- A role's places, and the roles in a set of places (the Place filter and
-- the country and place gates, `uid IN (... WHERE place_id IN ...)`).
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -469,6 +532,10 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
# A Python step: the first rebuild runs the rule in app/dupes.py.
(18, "roles.dup_of: one posting listed several times is one role, the others "
"copies of it", lambda conn: _add_dup_of(conn)),
# A Python step: the seed and the resolver are Python.
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
"places, resolved against a gazetteer; roles.teaser; the three seniority "
"sources", lambda conn: _add_places(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -592,6 +659,81 @@ def _add_dup_of(conn: sqlite3.Connection) -> None:
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
_V19 = """
ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT '';
CREATE TABLE places (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
key TEXT NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER REFERENCES places(id),
UNIQUE (kind, key)
);
CREATE TABLE place_aliases (
fold TEXT NOT NULL,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
UNIQUE (fold, place_id)
);
CREATE TABLE place_includes (
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (place_id, part_id)
);
CREATE TABLE role_places (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
position INTEGER NOT NULL,
text TEXT NOT NULL,
fold TEXT NOT NULL,
place_id INTEGER REFERENCES places(id),
url TEXT NOT NULL DEFAULT ''
);
"""
def _add_places(conn: sqlite3.Connection) -> None:
"""Migration 19's step. The tables and columns, the gazetteer seeded, and
every role's places: the `places` of its `raw` where a reader stated
them, else its `place` column cut at `;` (not `raw`'s, which a pull that
read the role's own page sends empty), each resolved once per distinct
text.
The level splits into its sources without changing: on a role extract
has read, `seniority` is the model's; on any other, it is the stated
level when `raw` states one in the vocabulary, else the title's."""
for statement in _statements(_V19):
conn.execute(statement)
gazetteer.seed(conn)
gaz = gazetteer.load(conn)
rows = conn.execute(
"SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall()
by_uid: list[tuple[str, list[str]]] = []
levels = []
for uid, raw, place, title, level, extracted_at in rows:
try:
given = json.loads(raw or "{}")
except ValueError:
given = {}
given = given if isinstance(given, dict) else {}
if found := gazetteer.texts({"places": given.get("places"), "place": place}):
by_uid.append((uid, found))
stated = str(given.get("seniority") or "")
stated = stated if stated in SENIORITIES else ""
if extracted_at is not None:
levels.append((level, stated, from_title(title) if level else "", uid))
elif stated:
levels.append(("", stated, from_title(title), uid))
elif level:
levels.append(("", "", level, uid))
written = gazetteer.write_many(conn, gaz, by_uid)
conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, "
"seniority_title = ? WHERE uid = ?", levels)
log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d",
len(gaz.places), written, len(by_uid), len(levels))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
@@ -659,6 +801,13 @@ def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
conn.execute("COMMIT")
def _create(conn: sqlite3.Connection) -> None:
"""A new file: `SCHEMA`, and the gazetteer seeded."""
for statement in _statements(SCHEMA):
conn.execute(statement)
gazetteer.seed(conn)
def initialise(conn: sqlite3.Connection) -> list[int]:
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
fresh = conn.execute(
@@ -668,7 +817,7 @@ def initialise(conn: sqlite3.Connection) -> list[int]:
if fresh:
# Atomic like a migration: a file half created would otherwise carry
# the marker table and be taken for one to migrate.
_apply(conn, LATEST, SCHEMA)
_apply(conn, LATEST, _create)
else:
applied = migrate(conn)
for statement in _statements(INDEXES):
+225 -55
View File
@@ -22,14 +22,14 @@ import numpy as np
from desk.models import Item
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
ScoredRole,
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
)
from app.seniority import from_title
from app.storage import schema
from app.storage import gazetteer, schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
@@ -101,7 +101,8 @@ _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
#: one, else the description's opening. A `SUBSTR` of the source column, not
#: the column itself, so a big `description` never leaves SQLite whole for a
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
_EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), "
"roles.teaser)")
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
@@ -177,23 +178,34 @@ def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
#: A role with no place the gazetteer resolved, over `roles r`.
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
"""One gate as `(empty, match, params)`: SQL that is true when the role's
column is empty, and SQL that is true when it matches, over `roles r`.
`params` belong to `match`; `empty` takes none."""
`params` belong to `match`; `empty` takes none.
`country` and `place` match over the gazetteer, as the feed's Place
filter does (`_places_sql`): a country gate is met by a role whose
`country` is one of its codes or that has a place in one of them, so
`DK` is met by `Copenhagen; London`. Its column is empty only when the
role has neither. A place gate's value is a place token, bare text a
`place:`."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "country":
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
codes=[v.upper() for v in values])
return f"(r.country = '' AND {_UNPLACED})", match, params
if gate.field == "place":
# Whole folded words, as the feed's Place filter: `lund` is not in
# `Billund`. A value is a place token, bare text a `place:`.
clauses = [places.match_sql(v, place="r.place_fold", country="r.country")
for v in values]
match = " OR ".join(sql for sql, _ in clauses) or "0"
return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps]
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
@@ -205,18 +217,54 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]:
"""The Place filter over `roles`: any of the tokens, each a `place_match`
call (`places.match_sql`). ValueError for a token that names nothing,
which would otherwise match no role without saying why."""
clauses, params = [], []
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
"""Any of the place tokens over `table`, as a join over the gazetteer: a
role with a place under one of the places the tokens name (`country:DK`
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
Malmö by any spelling), or whose `country` is one of their countries (a
role whose board gave a country and no place). `codes` are more values of
`country` to accept. A `place:` token the gazetteer does not know is
matched on whole words of the role's folded line (`places.match_sql`),
so a town nobody has entered yet is still found."""
ids: set[int] = set()
found = {c for c in codes if c}
clauses: list[str] = []
params: list[Any] = []
for token in tokens:
named = gaz.lookup(token)
if named is None:
sql, values = places.match_sql(token, place=f"{table}.place_fold",
country=f"{table}.country")
clauses.append(sql)
params += values
continue
under = gaz.under(named)
ids |= under
found |= gaz.codes(under)
if ids:
clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places "
"WHERE place_id IN (SELECT value FROM json_each(?)))")
params.insert(0, json.dumps(sorted(ids)))
if found:
clauses.insert(1 if ids else 0,
f"{table}.country IN (SELECT value FROM json_each(?))")
params.insert(1 if ids else 0, json.dumps(sorted(found)))
return f"({' OR '.join(clauses) or '0'})", params
def _known_places(tokens: Sequence[str]) -> None:
"""ValueError for a token that names nothing, which would otherwise match
no role without saying why."""
for token in tokens:
if not places.known(token):
raise ValueError(f"{token!r} is not a place this app knows")
sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country")
clauses.append(sql)
params += values
return f"({' OR '.join(clauses)})", params
def _text_rank(description: str, teaser: str) -> int:
"""What a role is embedded from, as a rank: 2 its description, 1 its
teaser, 0 its title alone."""
return 2 if description else 1 if teaser else 0
def _int_or_none(value: Any) -> int | None:
@@ -351,6 +399,8 @@ class Store:
self.path.parent.mkdir(parents=True, exist_ok=True)
#: What "now" is. Replaced by tests that need two writes a second apart.
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
#: The gazetteer as last read, with the `gazetteer.VERSION` it was read at.
self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
@@ -364,7 +414,7 @@ class Store:
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
registered on every one, since the place filter and the place gate
call it (`places.match_sql`)."""
call it for a town the gazetteer does not know (`_places_sql`)."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
places.register(conn)
@@ -400,6 +450,103 @@ class Store:
def now(self) -> str:
return _iso(self.clock())
# ------------------------------------------------------------- gazetteer
def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer:
"""The gazetteer, read again only when its tables have grown, which
another process may have done: one cheap query a call."""
version = tuple(conn.execute(gazetteer.VERSION).fetchone())
if self._gaz is None or self._gaz[0] != version:
self._gaz = (version, gazetteer.load(conn))
return self._gaz[1]
def gazetteer(self) -> places.Gazetteer:
with closing(self._connect()) as conn:
return self._gazetteer(conn)
def role_places(self, uid: str) -> list[RolePlace]:
"""One role's places in the host's order, a copy's after the role's own."""
return [RolePlace.from_row(r) for r in self._rows(
"""SELECT rp.position, rp.text, rp.place_id, rp.url,
COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind
FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id
WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""",
(uid,))]
def unresolved_places(self, open_only: bool = True,
limit: int | None = None) -> list[tuple[str, str, int]]:
"""`(fold, text, roles)` for every place text no place of the
gazetteer resolves, most roles first: the list a person works through
with `add_place_alias` and `add_place`. `text` is one spelling of the
fold. A text that names no place ("Remote", `places.placeless`) is
left out; with `open_only`, so are closed roles and copies."""
where = "rp.place_id IS NULL"
if open_only:
where += " AND r.closed_at IS NULL AND r.dup_of IS NULL"
rows = self._rows(
f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n
FROM role_places rp JOIN roles r ON r.uid = rp.uid
WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""")
found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows
if not places.placeless(r["fold"])]
return found[:limit] if limit is not None else found
def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int:
"""A place by id, or by a token that names exactly one place."""
if isinstance(place, int):
if place not in gaz.places:
raise ValueError(f"there is no place {place}")
return place
named = gaz.lookup(place) or frozenset()
if len(named) != 1:
raise ValueError(f"{place!r} names {len(named)} places, not one")
return next(iter(named))
def add_place_alias(self, text: str, place: int | str) -> int:
"""Write `text` as a spelling of `place` (an id, or a token such as
`place:copenhagen` or `country:DK` naming one place), and resolve again
every role with a place text that holds it. Returns how many roles now
resolve differently."""
folded = places.fold(text)
if not folded:
raise ValueError("an alias needs a letter or a digit")
with self.transaction() as conn:
place_id = self._place_id(self._gazetteer(conn), place)
conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
(folded, place_id))
return gazetteer.relink(conn, self._gazetteer(conn), folded)
def add_place(self, name: str, kind: str, parent: int | str,
aliases: Sequence[str] = ()) -> int:
"""Add a city or a region the gazetteer lacks, in `parent` (an id or a
token), written as `name` and each of `aliases`, and resolve again the
roles with a place text that holds one of them. Returns its id."""
if kind not in ("city", "region"):
raise ValueError(f"a place added by hand is a city or a region, not {kind!r}")
folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f))
if not folds:
raise ValueError("a place needs a name")
with self.transaction() as conn:
parent_id = self._place_id(self._gazetteer(conn), parent)
if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?",
(kind, folds[0])).fetchone():
raise ValueError(f"there is already a {kind} {name!r}")
place_id = int(conn.execute(
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
(kind, folds[0], name.strip(), parent_id)).lastrowid or 0)
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
[(f, place_id) for f in folds])
gaz = self._gazetteer(conn)
for folded in folds:
gazetteer.relink(conn, gaz, folded)
return place_id
def relink_places(self) -> int:
"""Resolve every role's own places again, after the gazetteer was
seeded anew. Returns how many roles now resolve differently."""
with self.transaction() as conn:
return gazetteer.relink(conn, self._gazetteer(conn))
# ---------------------------------------------------------------- roles
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
@@ -426,19 +573,32 @@ class Store:
alone for good, whatever the plugin's raw guess now says — see
`set_extracted` for the other half of the rule.
`seniority` is the title's level (`seniority.from_title`) until
extract has read the role and extract's from then on, as `function`
is. The classifier is called here rather than in a plugin, so a role
from any source gets it, the way `country` comes from `place` here
too. A level the board states in `raw` is a board field and wins over
both. `place_fold` is written with `place`, under the same keep-filled
rule.
`seniority` is the first filled of three levels. `seniority_model`
is extract's (`set_extracted`) and never written here.
`seniority_stated` is the host's word as `raw["seniority"]` carries
it, already in `SENIORITIES` (anything else is ''), and
`seniority_title` is `seniority.from_title`; both are rewritten every
pull. The classifier is called here rather than in a plugin, so a
role from any source gets it, the way `country` comes from `place`.
The role's places (`role_places`) are the texts of `raw["places"]`,
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
against the gazetteer; `place` is the line shown, the texts joined
when `raw` has none. Like `place`, a pull with no place leaves them
as they are. `place_fold` is written with `place`, under the same
keep-filled rule. `teaser` is kept-filled too.
A role is embedded from its description, else its teaser, else its
title (`app.jobs.embed.document`). When a pull brings a better one of
these than the role had, its vector is dropped so the next `embed`
reads it.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
raw = item.raw or {}
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
place_texts = gazetteer.texts(raw)
place = str(raw.get("place") or "") or "; ".join(place_texts)
stamp = self.now()
published = _iso(item.published_at)
row: dict[str, Any] = {
@@ -463,12 +623,16 @@ class Store:
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row["place"] = place
row["country"] = row["country"] or country_of(place)
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
stated = str(raw.get("seniority") or "")
row["seniority"] = stated or from_title(item.title)
row["seniority_stated"] = stated if stated in SENIORITIES else ""
row["seniority_title"] = from_title(item.title)
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
row["teaser"] = str(raw.get("teaser") or "")
row["place_fold"] = places.fold(row["place"])
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
@@ -476,13 +640,13 @@ class Store:
row["salary_text"] = str(raw.get("salary_text") or "")
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
"feed_text", "original_url", "teaser", *_RAW_TEXT)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text")]
"salary_text", "seniority_stated", "seniority_title")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -493,27 +657,29 @@ class Store:
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
"ELSE roles.function END")
# The title's level follows the title until extract has read the
# role, and is extract's from then on; a board's stated level wins.
updates.append("seniority=excluded.seniority" if stated else
"seniority=CASE WHEN roles.extracted_at IS NULL "
"THEN excluded.seniority ELSE roles.seniority END")
# The model's level, else the host's, else the title's.
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
updates.append("place_fold=CASE WHEN excluded.place != '' "
"THEN excluded.place_fold ELSE roles.place_fold END")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
with self.transaction() as conn:
before = conn.execute("SELECT description FROM roles WHERE uid=?",
before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?",
(row["uid"],)).fetchone()
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
# A role listed without its text is embedded from its title, and
# `embed` encodes a role once. When the text arrives on a later
# pull, as it does for a host that states it on the role's own
# page, the vector goes, so the next `embed` reads the text.
if before is not None and not before["description"] and row["description"]:
if place_texts:
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
# `embed` encodes a role once, from the best text it has. When a
# later pull brings a better one, as a host that states the text
# on the role's own page does, the vector goes, so the next
# `embed` reads it.
if before is not None and _text_rank(row["description"] or before["description"],
row["teaser"] or before["teaser"]) \
> _text_rank(before["description"], before["teaser"]):
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
return before is None
@@ -533,6 +699,7 @@ class Store:
role = ScoredRole.from_row(row)
role.tags = self._tags_of(uid)
role.copies = self._copies_of(uid)
role.places = self.role_places(uid)
return role
def _copies_of(self, uid: str) -> list[RoleCopy]:
@@ -685,7 +852,7 @@ class Store:
`company_keys` keeps those exact companies. `places` keeps a role any
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
`place:malmo`), matched on whole folded words; a token that names
`place:malmo`), over the gazetteer (`_places_sql`); a token that names
nothing is a ValueError. `seniority` keeps the roles at any of those
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
with no level, which is most of them. An empty list is no filter.
@@ -934,8 +1101,8 @@ class Store:
else:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
if places:
# Last, so the cheaper terms have cut the rows it is called on.
clause, values = _places_sql(places)
_known_places(places)
clause, values = _places_sql(places, self.gazetteer())
where.append(clause)
params += values
sql = f"FROM roles {_WITH_VOTES}"
@@ -1087,9 +1254,9 @@ class Store:
non-empty answer overwrites it outright rather than only filling an
empty column — see `upsert_role`'s docstring for the other half of
the rule that then keeps a later pull from putting the guess back.
`seniority` is the other: `upsert_role` fills it from the title, so a
non-empty answer replaces it, unless the board stated a level in
`raw`."""
`seniority` is the other: a non-empty answer is `seniority_model`,
which comes before the stated and the title's level, so it becomes
`seniority` too."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1103,8 +1270,8 @@ class Store:
if function:
sets["function"] = function
level = str(fields.get("seniority") or "")
if level and not json.loads(row["raw"] or "{}").get("seniority"):
sets["seniority"] = level
if level:
sets["seniority_model"] = sets["seniority"] = level
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
@@ -1112,10 +1279,12 @@ class Store:
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets)
filled = len(sets) - ("seniority" in sets)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
gazetteer.write(conn, self._gazetteer(conn), uid,
gazetteer.texts({"place": sets["place"]}))
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
@@ -1375,8 +1544,9 @@ class Store:
as a column so a gated role keeps its row and reason."""
columns: list[str] = []
params: list[Any] = []
gaz = self.gazetteer()
for n, gate in enumerate(gates):
empty, match, values = _gate_sql(gate)
empty, match, values = _gate_sql(gate, gaz)
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values