Places as a gazetteer, role_places, teaser and three seniority sources
Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
bbd7522163
10 files changed
+1437
-98
No files matched your search
@@ -3,6 +3,8 @@
|
||||
schema.py the tables, and the numbered migrations that bring an older
|
||||
file to the current shape
|
||||
store.py `Store`, every SQL statement in the application
|
||||
gazetteer.py the places tables: seeded from the code, read back, and a
|
||||
role's places written and relinked
|
||||
|
||||
Written for roledesk rather than imported from newsdesk: `desk` has no storage
|
||||
package on purpose, and the connection plumbing here is a deliberate copy.
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""The gazetteer's tables: seeding them from the code, reading them back, and
|
||||
writing a role's places.
|
||||
|
||||
`places`, `place_aliases` and `place_includes` hold the places
|
||||
(`app.places.seed` on a new or migrated file, and what a person adds later).
|
||||
`role_places` holds one row per place of a role: the host's own text, its
|
||||
fold, and the place it resolves to, NULL for a text the gazetteer does not
|
||||
know. A text that names several places ("Malmö/Lund") is one row per place,
|
||||
each with the same `position` and `text`. A row with a `url` is the place of
|
||||
a copy folded into the role (`app/dupes.py`) and is not the role's own: only
|
||||
own rows (`url = ''`) are rewritten here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
from app import places
|
||||
|
||||
#: What changes when the gazetteer does: rows are only ever added.
|
||||
VERSION = ("SELECT (SELECT MAX(rowid) FROM places), (SELECT MAX(rowid) FROM place_aliases), "
|
||||
"(SELECT COUNT(*) FROM place_includes)")
|
||||
|
||||
|
||||
def seed(conn: sqlite3.Connection) -> int:
|
||||
"""Insert what `places.seed` describes, leaving rows already there alone.
|
||||
Returns how many places were added."""
|
||||
seeds, includes = places.seed()
|
||||
ids = {(kind, key): place_id for place_id, kind, key in
|
||||
conn.execute("SELECT id, kind, key FROM places")}
|
||||
added = 0
|
||||
for s in seeds:
|
||||
if (s.kind, s.key) not in ids:
|
||||
parent = ids[s.parent] if s.parent else None
|
||||
ids[(s.kind, s.key)] = conn.execute(
|
||||
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
|
||||
(s.kind, s.key, s.name, parent)).lastrowid
|
||||
added += 1
|
||||
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
||||
[(a, ids[(s.kind, s.key)]) for a in s.aliases])
|
||||
conn.executemany("INSERT OR IGNORE INTO place_includes (place_id, part_id) VALUES (?, ?)",
|
||||
[(ids[whole], ids[part]) for whole, part in includes])
|
||||
return added
|
||||
|
||||
|
||||
def load(conn: sqlite3.Connection) -> places.Gazetteer:
|
||||
return places.Gazetteer(
|
||||
[tuple(r) for r in conn.execute("SELECT id, kind, key, name, parent_id FROM places")],
|
||||
[tuple(r) for r in conn.execute("SELECT fold, place_id FROM place_aliases")],
|
||||
[tuple(r) for r in conn.execute("SELECT place_id, part_id FROM place_includes")])
|
||||
|
||||
|
||||
def texts(raw: Mapping[str, Any]) -> list[str]:
|
||||
"""A role's place texts from `Item.raw`: the `text` of each of `places`
|
||||
(`[{"text": ..., "remote": ...}]`, the reader contract), else `place` split
|
||||
at `;`, which is how the older readers join several places into one line.
|
||||
Blank and repeated texts are dropped."""
|
||||
given = raw.get("places")
|
||||
if isinstance(given, list) and given:
|
||||
found = [str((p.get("text") if isinstance(p, Mapping) else p) or "") for p in given]
|
||||
else:
|
||||
found = str(raw.get("place") or "").split(";")
|
||||
return list(dict.fromkeys(t.strip() for t in found if t and t.strip()))
|
||||
|
||||
|
||||
def rows(gaz: places.Gazetteer, uid: str, place_texts: Sequence[str]
|
||||
) -> list[tuple[str, int, str, str, int | None]]:
|
||||
"""`(uid, position, text, fold, place_id)` for each place of the texts."""
|
||||
found: list[tuple[str, int, str, str, int | None]] = []
|
||||
for position, text in enumerate(place_texts):
|
||||
folded = places.fold(text)
|
||||
found += [(uid, position, text, folded, place_id)
|
||||
for place_id in (gaz.resolve(text) or (None,))]
|
||||
return found
|
||||
|
||||
|
||||
_INSERT = "INSERT INTO role_places (uid, position, text, fold, place_id) VALUES (?, ?, ?, ?, ?)"
|
||||
|
||||
|
||||
def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
|
||||
place_texts: Sequence[str]) -> None:
|
||||
"""Replace one role's own places with these texts, resolved."""
|
||||
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
|
||||
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
||||
|
||||
|
||||
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
||||
"""`write` for many roles in one statement each way. Returns rows written."""
|
||||
batch = list(by_uid)
|
||||
conn.executemany("DELETE FROM role_places WHERE uid = ? AND url = ''",
|
||||
[(uid,) for uid, _ in batch])
|
||||
new = [row for uid, found in batch for row in rows(gaz, uid, found)]
|
||||
conn.executemany(_INSERT, new)
|
||||
return len(new)
|
||||
|
||||
|
||||
def relink(conn: sqlite3.Connection, gaz: places.Gazetteer, words: str | None = None) -> int:
|
||||
"""Resolve the own places of every role again, or only of the roles with a
|
||||
place text that holds `words` (a fold) as whole words, after an alias or a
|
||||
place was added. Returns how many roles now resolve differently."""
|
||||
sql = "SELECT uid, position, text, place_id FROM role_places WHERE url = ''"
|
||||
params: list[Any] = []
|
||||
if words:
|
||||
sql += " AND ' ' || fold || ' ' LIKE ? ESCAPE '\\'"
|
||||
escaped = words.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
params.append(f"% {escaped} %")
|
||||
sql = ("SELECT uid, position, text, place_id FROM role_places WHERE url = '' AND uid IN "
|
||||
f"(SELECT uid FROM ({sql}))")
|
||||
before: dict[str, dict[tuple[int, str], set[int | None]]] = {}
|
||||
for uid, position, text, place_id in conn.execute(sql + " ORDER BY uid, position", params):
|
||||
before.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
|
||||
changed = []
|
||||
for uid, held in before.items():
|
||||
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in held}
|
||||
if now != held:
|
||||
changed.append((uid, [text for _, text in sorted(held)]))
|
||||
write_many(conn, gaz, changed)
|
||||
return len(changed)
|
||||
+156
-7
@@ -18,8 +18,9 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import country_of, posted_at, posted_host
|
||||
from app.models import SENIORITIES, country_of, posted_at, posted_host
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -44,8 +45,9 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- require the country: "UK | Remote" is not remote from Berlin.
|
||||
place TEXT NOT NULL DEFAULT '',
|
||||
place_key TEXT NOT NULL DEFAULT '',
|
||||
-- places.fold(place): what the Place filter and the place gate match
|
||||
-- whole words against, through the place_match function (app/places.py).
|
||||
-- places.fold(place): what a place: token the gazetteer does not know is
|
||||
-- matched against as whole words, through place_match (app/places.py).
|
||||
-- The filter and the gates otherwise read role_places.
|
||||
place_fold TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
@@ -53,7 +55,8 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
-- seniority.from_title(title) at ingest, until extract reads the posting.
|
||||
-- The level shown and filtered on: the first filled of seniority_model,
|
||||
-- seniority_stated and seniority_title (the last columns), written with them.
|
||||
seniority TEXT NOT NULL DEFAULT '',
|
||||
function TEXT NOT NULL DEFAULT '',
|
||||
-- Whether the posting says it sponsors a work permit, and the languages
|
||||
@@ -69,7 +72,7 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- stored the role; posted_at (app.models.posted_at) is the earlier of
|
||||
-- first_seen and published_at, trusted only for a host known to report a
|
||||
-- creation date rather than one that moves on an edit
|
||||
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
|
||||
-- (app.models.trusted_posted) — recency and the feed's newest order
|
||||
-- read posted_at, first_seen stays "when the app found it". closed_at is
|
||||
-- set when a board fetched in full no longer lists the role, and cleared
|
||||
-- if it returns.
|
||||
@@ -100,7 +103,62 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- precedes, which the migration tests do.
|
||||
dup_of TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
salary_text TEXT NOT NULL DEFAULT ''
|
||||
salary_text TEXT NOT NULL DEFAULT '',
|
||||
teaser TEXT NOT NULL DEFAULT '',
|
||||
seniority_model TEXT NOT NULL DEFAULT '',
|
||||
seniority_stated TEXT NOT NULL DEFAULT '',
|
||||
seniority_title TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
-- roles.teaser is the short text a list carries, kept until the full text
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
-- three levels behind roles.seniority: the model's (extract), the host's own
|
||||
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
|
||||
-- among the last columns: SQLite cannot drop a column a comment precedes,
|
||||
-- which the migration tests do.
|
||||
|
||||
-- The gazetteer: a place is a continent, a country (key the ISO code), a
|
||||
-- region or a city, in the place parent_id names. key is unique per kind:
|
||||
-- a region's key (app.places.REGIONS), a town's group. Seeded from the
|
||||
-- dictionaries in app/models.py and app/places.py (app.places.seed).
|
||||
CREATE TABLE IF NOT EXISTS places (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
|
||||
key TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
parent_id INTEGER REFERENCES places(id),
|
||||
UNIQUE (kind, key)
|
||||
);
|
||||
|
||||
-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn
|
||||
-- and copenhagen are Copenhagen. One fold may name several places
|
||||
-- (Luxembourg the town and the country).
|
||||
CREATE TABLE IF NOT EXISTS place_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
UNIQUE (fold, place_id)
|
||||
);
|
||||
|
||||
-- What a place holds beyond its children: Øresund holds the Capital Region
|
||||
-- of Denmark and Skåne, which lie in two countries.
|
||||
CREATE TABLE IF NOT EXISTS place_includes (
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (place_id, part_id)
|
||||
);
|
||||
|
||||
-- One row per place of a role (app/storage/gazetteer.py): the host's text in
|
||||
-- the host's order (position), its fold, and the place it resolves to, NULL
|
||||
-- while the gazetteer does not know it. A text naming several places is a
|
||||
-- row for each. url is the link of a copy folded into this role, '' for
|
||||
-- the role's own places.
|
||||
CREATE TABLE IF NOT EXISTS role_places (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER REFERENCES places(id),
|
||||
url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- The open vocabulary summarise reads off a posting: the role family, the
|
||||
@@ -303,6 +361,11 @@ CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
|
||||
-- Partial: all but about 3% of roles are no copy.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
ON roles(dup_of) WHERE dup_of IS NOT NULL;
|
||||
-- A role's places, and the roles in a set of places (the Place filter and
|
||||
-- the country and place gates, `uid IN (... WHERE place_id IN ...)`).
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -469,6 +532,10 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
# A Python step: the first rebuild runs the rule in app/dupes.py.
|
||||
(18, "roles.dup_of: one posting listed several times is one role, the others "
|
||||
"copies of it", lambda conn: _add_dup_of(conn)),
|
||||
# A Python step: the seed and the resolver are Python.
|
||||
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
|
||||
"places, resolved against a gazetteer; roles.teaser; the three seniority "
|
||||
"sources", lambda conn: _add_places(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -592,6 +659,81 @@ def _add_dup_of(conn: sqlite3.Connection) -> None:
|
||||
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
|
||||
|
||||
|
||||
_V19 = """
|
||||
ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT '';
|
||||
CREATE TABLE places (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
|
||||
key TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
parent_id INTEGER REFERENCES places(id),
|
||||
UNIQUE (kind, key)
|
||||
);
|
||||
CREATE TABLE place_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
UNIQUE (fold, place_id)
|
||||
);
|
||||
CREATE TABLE place_includes (
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (place_id, part_id)
|
||||
);
|
||||
CREATE TABLE role_places (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER REFERENCES places(id),
|
||||
url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def _add_places(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 19's step. The tables and columns, the gazetteer seeded, and
|
||||
every role's places: the `places` of its `raw` where a reader stated
|
||||
them, else its `place` column cut at `;` (not `raw`'s, which a pull that
|
||||
read the role's own page sends empty), each resolved once per distinct
|
||||
text.
|
||||
|
||||
The level splits into its sources without changing: on a role extract
|
||||
has read, `seniority` is the model's; on any other, it is the stated
|
||||
level when `raw` states one in the vocabulary, else the title's."""
|
||||
for statement in _statements(_V19):
|
||||
conn.execute(statement)
|
||||
gazetteer.seed(conn)
|
||||
gaz = gazetteer.load(conn)
|
||||
rows = conn.execute(
|
||||
"SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall()
|
||||
by_uid: list[tuple[str, list[str]]] = []
|
||||
levels = []
|
||||
for uid, raw, place, title, level, extracted_at in rows:
|
||||
try:
|
||||
given = json.loads(raw or "{}")
|
||||
except ValueError:
|
||||
given = {}
|
||||
given = given if isinstance(given, dict) else {}
|
||||
if found := gazetteer.texts({"places": given.get("places"), "place": place}):
|
||||
by_uid.append((uid, found))
|
||||
stated = str(given.get("seniority") or "")
|
||||
stated = stated if stated in SENIORITIES else ""
|
||||
if extracted_at is not None:
|
||||
levels.append((level, stated, from_title(title) if level else "", uid))
|
||||
elif stated:
|
||||
levels.append(("", stated, from_title(title), uid))
|
||||
elif level:
|
||||
levels.append(("", "", level, uid))
|
||||
written = gazetteer.write_many(conn, gaz, by_uid)
|
||||
conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, "
|
||||
"seniority_title = ? WHERE uid = ?", levels)
|
||||
log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d",
|
||||
len(gaz.places), written, len(by_uid), len(levels))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
@@ -659,6 +801,13 @@ def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def _create(conn: sqlite3.Connection) -> None:
|
||||
"""A new file: `SCHEMA`, and the gazetteer seeded."""
|
||||
for statement in _statements(SCHEMA):
|
||||
conn.execute(statement)
|
||||
gazetteer.seed(conn)
|
||||
|
||||
|
||||
def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
||||
fresh = conn.execute(
|
||||
@@ -668,7 +817,7 @@ def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
if fresh:
|
||||
# Atomic like a migration: a file half created would otherwise carry
|
||||
# the marker table and be taken for one to migrate.
|
||||
_apply(conn, LATEST, SCHEMA)
|
||||
_apply(conn, LATEST, _create)
|
||||
else:
|
||||
applied = migrate(conn)
|
||||
for statement in _statements(INDEXES):
|
||||
|
||||
+225
-55
@@ -22,14 +22,14 @@ import numpy as np
|
||||
from desk.models import Item
|
||||
from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
|
||||
ScoredRole,
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
||||
RoleRow, ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import schema
|
||||
from app.storage import gazetteer, schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
#: empty value never overwrites a filled column, so an upsert from a board that
|
||||
@@ -101,7 +101,8 @@ _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
||||
#: one, else the description's opening. A `SUBSTR` of the source column, not
|
||||
#: the column itself, so a big `description` never leaves SQLite whole for a
|
||||
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
|
||||
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
|
||||
_EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), "
|
||||
"roles.teaser)")
|
||||
|
||||
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
|
||||
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
|
||||
@@ -177,23 +178,34 @@ def unpack(blob: bytes) -> np.ndarray:
|
||||
return np.frombuffer(blob, dtype=_DTYPE)
|
||||
|
||||
|
||||
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
||||
#: A role with no place the gazetteer resolved, over `roles r`.
|
||||
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
|
||||
|
||||
|
||||
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
"""One gate as `(empty, match, params)`: SQL that is true when the role's
|
||||
column is empty, and SQL that is true when it matches, over `roles r`.
|
||||
`params` belong to `match`; `empty` takes none."""
|
||||
`params` belong to `match`; `empty` takes none.
|
||||
|
||||
`country` and `place` match over the gazetteer, as the feed's Place
|
||||
filter does (`_places_sql`): a country gate is met by a role whose
|
||||
`country` is one of its codes or that has a place in one of them, so
|
||||
`DK` is met by `Copenhagen; London`. Its column is empty only when the
|
||||
role has neither. A place gate's value is a place token, bare text a
|
||||
`place:`."""
|
||||
column = f"r.{gate.field}"
|
||||
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
||||
marks = ", ".join("?" for _ in values) or "NULL"
|
||||
if gate.field == "salary_min":
|
||||
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
||||
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
||||
if gate.field == "country":
|
||||
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
|
||||
codes=[v.upper() for v in values])
|
||||
return f"(r.country = '' AND {_UNPLACED})", match, params
|
||||
if gate.field == "place":
|
||||
# Whole folded words, as the feed's Place filter: `lund` is not in
|
||||
# `Billund`. A value is a place token, bare text a `place:`.
|
||||
clauses = [places.match_sql(v, place="r.place_fold", country="r.country")
|
||||
for v in values]
|
||||
match = " OR ".join(sql for sql, _ in clauses) or "0"
|
||||
return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps]
|
||||
match, params = _places_sql(values, gaz, table="r")
|
||||
return "r.place = ''", match, params
|
||||
if gate.field == "company":
|
||||
keys = [company_key(v) for v in values]
|
||||
return ("r.company_key = ''",
|
||||
@@ -205,18 +217,54 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||
|
||||
|
||||
def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]:
|
||||
"""The Place filter over `roles`: any of the tokens, each a `place_match`
|
||||
call (`places.match_sql`). ValueError for a token that names nothing,
|
||||
which would otherwise match no role without saying why."""
|
||||
clauses, params = [], []
|
||||
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
|
||||
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
|
||||
"""Any of the place tokens over `table`, as a join over the gazetteer: a
|
||||
role with a place under one of the places the tokens name (`country:DK`
|
||||
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
|
||||
Malmö by any spelling), or whose `country` is one of their countries (a
|
||||
role whose board gave a country and no place). `codes` are more values of
|
||||
`country` to accept. A `place:` token the gazetteer does not know is
|
||||
matched on whole words of the role's folded line (`places.match_sql`),
|
||||
so a town nobody has entered yet is still found."""
|
||||
ids: set[int] = set()
|
||||
found = {c for c in codes if c}
|
||||
clauses: list[str] = []
|
||||
params: list[Any] = []
|
||||
for token in tokens:
|
||||
named = gaz.lookup(token)
|
||||
if named is None:
|
||||
sql, values = places.match_sql(token, place=f"{table}.place_fold",
|
||||
country=f"{table}.country")
|
||||
clauses.append(sql)
|
||||
params += values
|
||||
continue
|
||||
under = gaz.under(named)
|
||||
ids |= under
|
||||
found |= gaz.codes(under)
|
||||
if ids:
|
||||
clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places "
|
||||
"WHERE place_id IN (SELECT value FROM json_each(?)))")
|
||||
params.insert(0, json.dumps(sorted(ids)))
|
||||
if found:
|
||||
clauses.insert(1 if ids else 0,
|
||||
f"{table}.country IN (SELECT value FROM json_each(?))")
|
||||
params.insert(1 if ids else 0, json.dumps(sorted(found)))
|
||||
return f"({' OR '.join(clauses) or '0'})", params
|
||||
|
||||
|
||||
def _known_places(tokens: Sequence[str]) -> None:
|
||||
"""ValueError for a token that names nothing, which would otherwise match
|
||||
no role without saying why."""
|
||||
for token in tokens:
|
||||
if not places.known(token):
|
||||
raise ValueError(f"{token!r} is not a place this app knows")
|
||||
sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country")
|
||||
clauses.append(sql)
|
||||
params += values
|
||||
return f"({' OR '.join(clauses)})", params
|
||||
|
||||
|
||||
def _text_rank(description: str, teaser: str) -> int:
|
||||
"""What a role is embedded from, as a rank: 2 its description, 1 its
|
||||
teaser, 0 its title alone."""
|
||||
return 2 if description else 1 if teaser else 0
|
||||
|
||||
|
||||
def _int_or_none(value: Any) -> int | None:
|
||||
@@ -351,6 +399,8 @@ class Store:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
#: What "now" is. Replaced by tests that need two writes a second apart.
|
||||
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
||||
#: The gazetteer as last read, with the `gazetteer.VERSION` it was read at.
|
||||
self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None
|
||||
with self.transaction() as conn:
|
||||
# Persistent: recorded in the file, inherited by every connection.
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
@@ -364,7 +414,7 @@ class Store:
|
||||
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
||||
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
|
||||
registered on every one, since the place filter and the place gate
|
||||
call it (`places.match_sql`)."""
|
||||
call it for a town the gazetteer does not know (`_places_sql`)."""
|
||||
conn = sqlite3.connect(self.path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
places.register(conn)
|
||||
@@ -400,6 +450,103 @@ class Store:
|
||||
def now(self) -> str:
|
||||
return _iso(self.clock())
|
||||
|
||||
# ------------------------------------------------------------- gazetteer
|
||||
|
||||
def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer:
|
||||
"""The gazetteer, read again only when its tables have grown, which
|
||||
another process may have done: one cheap query a call."""
|
||||
version = tuple(conn.execute(gazetteer.VERSION).fetchone())
|
||||
if self._gaz is None or self._gaz[0] != version:
|
||||
self._gaz = (version, gazetteer.load(conn))
|
||||
return self._gaz[1]
|
||||
|
||||
def gazetteer(self) -> places.Gazetteer:
|
||||
with closing(self._connect()) as conn:
|
||||
return self._gazetteer(conn)
|
||||
|
||||
def role_places(self, uid: str) -> list[RolePlace]:
|
||||
"""One role's places in the host's order, a copy's after the role's own."""
|
||||
return [RolePlace.from_row(r) for r in self._rows(
|
||||
"""SELECT rp.position, rp.text, rp.place_id, rp.url,
|
||||
COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind
|
||||
FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id
|
||||
WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""",
|
||||
(uid,))]
|
||||
|
||||
def unresolved_places(self, open_only: bool = True,
|
||||
limit: int | None = None) -> list[tuple[str, str, int]]:
|
||||
"""`(fold, text, roles)` for every place text no place of the
|
||||
gazetteer resolves, most roles first: the list a person works through
|
||||
with `add_place_alias` and `add_place`. `text` is one spelling of the
|
||||
fold. A text that names no place ("Remote", `places.placeless`) is
|
||||
left out; with `open_only`, so are closed roles and copies."""
|
||||
where = "rp.place_id IS NULL"
|
||||
if open_only:
|
||||
where += " AND r.closed_at IS NULL AND r.dup_of IS NULL"
|
||||
rows = self._rows(
|
||||
f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n
|
||||
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
||||
WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""")
|
||||
found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows
|
||||
if not places.placeless(r["fold"])]
|
||||
return found[:limit] if limit is not None else found
|
||||
|
||||
def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int:
|
||||
"""A place by id, or by a token that names exactly one place."""
|
||||
if isinstance(place, int):
|
||||
if place not in gaz.places:
|
||||
raise ValueError(f"there is no place {place}")
|
||||
return place
|
||||
named = gaz.lookup(place) or frozenset()
|
||||
if len(named) != 1:
|
||||
raise ValueError(f"{place!r} names {len(named)} places, not one")
|
||||
return next(iter(named))
|
||||
|
||||
def add_place_alias(self, text: str, place: int | str) -> int:
|
||||
"""Write `text` as a spelling of `place` (an id, or a token such as
|
||||
`place:copenhagen` or `country:DK` naming one place), and resolve again
|
||||
every role with a place text that holds it. Returns how many roles now
|
||||
resolve differently."""
|
||||
folded = places.fold(text)
|
||||
if not folded:
|
||||
raise ValueError("an alias needs a letter or a digit")
|
||||
with self.transaction() as conn:
|
||||
place_id = self._place_id(self._gazetteer(conn), place)
|
||||
conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
||||
(folded, place_id))
|
||||
return gazetteer.relink(conn, self._gazetteer(conn), folded)
|
||||
|
||||
def add_place(self, name: str, kind: str, parent: int | str,
|
||||
aliases: Sequence[str] = ()) -> int:
|
||||
"""Add a city or a region the gazetteer lacks, in `parent` (an id or a
|
||||
token), written as `name` and each of `aliases`, and resolve again the
|
||||
roles with a place text that holds one of them. Returns its id."""
|
||||
if kind not in ("city", "region"):
|
||||
raise ValueError(f"a place added by hand is a city or a region, not {kind!r}")
|
||||
folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f))
|
||||
if not folds:
|
||||
raise ValueError("a place needs a name")
|
||||
with self.transaction() as conn:
|
||||
parent_id = self._place_id(self._gazetteer(conn), parent)
|
||||
if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?",
|
||||
(kind, folds[0])).fetchone():
|
||||
raise ValueError(f"there is already a {kind} {name!r}")
|
||||
place_id = int(conn.execute(
|
||||
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
|
||||
(kind, folds[0], name.strip(), parent_id)).lastrowid or 0)
|
||||
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
||||
[(f, place_id) for f in folds])
|
||||
gaz = self._gazetteer(conn)
|
||||
for folded in folds:
|
||||
gazetteer.relink(conn, gaz, folded)
|
||||
return place_id
|
||||
|
||||
def relink_places(self) -> int:
|
||||
"""Resolve every role's own places again, after the gazetteer was
|
||||
seeded anew. Returns how many roles now resolve differently."""
|
||||
with self.transaction() as conn:
|
||||
return gazetteer.relink(conn, self._gazetteer(conn))
|
||||
|
||||
# ---------------------------------------------------------------- roles
|
||||
|
||||
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
|
||||
@@ -426,19 +573,32 @@ class Store:
|
||||
alone for good, whatever the plugin's raw guess now says — see
|
||||
`set_extracted` for the other half of the rule.
|
||||
|
||||
`seniority` is the title's level (`seniority.from_title`) until
|
||||
extract has read the role and extract's from then on, as `function`
|
||||
is. The classifier is called here rather than in a plugin, so a role
|
||||
from any source gets it, the way `country` comes from `place` here
|
||||
too. A level the board states in `raw` is a board field and wins over
|
||||
both. `place_fold` is written with `place`, under the same keep-filled
|
||||
rule.
|
||||
`seniority` is the first filled of three levels. `seniority_model`
|
||||
is extract's (`set_extracted`) and never written here.
|
||||
`seniority_stated` is the host's word as `raw["seniority"]` carries
|
||||
it, already in `SENIORITIES` (anything else is ''), and
|
||||
`seniority_title` is `seniority.from_title`; both are rewritten every
|
||||
pull. The classifier is called here rather than in a plugin, so a
|
||||
role from any source gets it, the way `country` comes from `place`.
|
||||
|
||||
The role's places (`role_places`) are the texts of `raw["places"]`,
|
||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
|
||||
against the gazetteer; `place` is the line shown, the texts joined
|
||||
when `raw` has none. Like `place`, a pull with no place leaves them
|
||||
as they are. `place_fold` is written with `place`, under the same
|
||||
keep-filled rule. `teaser` is kept-filled too.
|
||||
|
||||
A role is embedded from its description, else its teaser, else its
|
||||
title (`app.jobs.embed.document`). When a pull brings a better one of
|
||||
these than the role had, its vector is dropped so the next `embed`
|
||||
reads it.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
raw = item.raw or {}
|
||||
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
||||
place = str(raw.get("place") or "")
|
||||
place_texts = gazetteer.texts(raw)
|
||||
place = str(raw.get("place") or "") or "; ".join(place_texts)
|
||||
stamp = self.now()
|
||||
published = _iso(item.published_at)
|
||||
row: dict[str, Any] = {
|
||||
@@ -463,12 +623,16 @@ class Store:
|
||||
"created_at": stamp,
|
||||
}
|
||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||
row["place"] = place
|
||||
row["country"] = row["country"] or country_of(place)
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
stated = str(raw.get("seniority") or "")
|
||||
row["seniority"] = stated or from_title(item.title)
|
||||
row["seniority_stated"] = stated if stated in SENIORITIES else ""
|
||||
row["seniority_title"] = from_title(item.title)
|
||||
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
|
||||
row["teaser"] = str(raw.get("teaser") or "")
|
||||
row["place_fold"] = places.fold(row["place"])
|
||||
# A board's own pay line, one per pull like title or published_at —
|
||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||
@@ -476,13 +640,13 @@ class Store:
|
||||
row["salary_text"] = str(raw.get("salary_text") or "")
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", *_RAW_TEXT)
|
||||
"feed_text", "original_url", "teaser", *_RAW_TEXT)
|
||||
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
||||
"salary_text")]
|
||||
"salary_text", "seniority_stated", "seniority_title")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
@@ -493,27 +657,29 @@ class Store:
|
||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||
"ELSE roles.function END")
|
||||
# The title's level follows the title until extract has read the
|
||||
# role, and is extract's from then on; a board's stated level wins.
|
||||
updates.append("seniority=excluded.seniority" if stated else
|
||||
"seniority=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN excluded.seniority ELSE roles.seniority END")
|
||||
# The model's level, else the host's, else the title's.
|
||||
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
|
||||
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
|
||||
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
||||
"THEN excluded.place_fold ELSE roles.place_fold END")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
with self.transaction() as conn:
|
||||
before = conn.execute("SELECT description FROM roles WHERE uid=?",
|
||||
before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?",
|
||||
(row["uid"],)).fetchone()
|
||||
conn.execute(
|
||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||
# A role listed without its text is embedded from its title, and
|
||||
# `embed` encodes a role once. When the text arrives on a later
|
||||
# pull, as it does for a host that states it on the role's own
|
||||
# page, the vector goes, so the next `embed` reads the text.
|
||||
if before is not None and not before["description"] and row["description"]:
|
||||
if place_texts:
|
||||
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
|
||||
# `embed` encodes a role once, from the best text it has. When a
|
||||
# later pull brings a better one, as a host that states the text
|
||||
# on the role's own page does, the vector goes, so the next
|
||||
# `embed` reads it.
|
||||
if before is not None and _text_rank(row["description"] or before["description"],
|
||||
row["teaser"] or before["teaser"]) \
|
||||
> _text_rank(before["description"], before["teaser"]):
|
||||
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
|
||||
return before is None
|
||||
|
||||
@@ -533,6 +699,7 @@ class Store:
|
||||
role = ScoredRole.from_row(row)
|
||||
role.tags = self._tags_of(uid)
|
||||
role.copies = self._copies_of(uid)
|
||||
role.places = self.role_places(uid)
|
||||
return role
|
||||
|
||||
def _copies_of(self, uid: str) -> list[RoleCopy]:
|
||||
@@ -685,7 +852,7 @@ class Store:
|
||||
|
||||
`company_keys` keeps those exact companies. `places` keeps a role any
|
||||
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
|
||||
`place:malmo`), matched on whole folded words; a token that names
|
||||
`place:malmo`), over the gazetteer (`_places_sql`); a token that names
|
||||
nothing is a ValueError. `seniority` keeps the roles at any of those
|
||||
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
|
||||
with no level, which is most of them. An empty list is no filter.
|
||||
@@ -934,8 +1101,8 @@ class Store:
|
||||
else:
|
||||
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
|
||||
if places:
|
||||
# Last, so the cheaper terms have cut the rows it is called on.
|
||||
clause, values = _places_sql(places)
|
||||
_known_places(places)
|
||||
clause, values = _places_sql(places, self.gazetteer())
|
||||
where.append(clause)
|
||||
params += values
|
||||
sql = f"FROM roles {_WITH_VOTES}"
|
||||
@@ -1087,9 +1254,9 @@ class Store:
|
||||
non-empty answer overwrites it outright rather than only filling an
|
||||
empty column — see `upsert_role`'s docstring for the other half of
|
||||
the rule that then keeps a later pull from putting the guess back.
|
||||
`seniority` is the other: `upsert_role` fills it from the title, so a
|
||||
non-empty answer replaces it, unless the board stated a level in
|
||||
`raw`."""
|
||||
`seniority` is the other: a non-empty answer is `seniority_model`,
|
||||
which comes before the stated and the title's level, so it becomes
|
||||
`seniority` too."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
@@ -1103,8 +1270,8 @@ class Store:
|
||||
if function:
|
||||
sets["function"] = function
|
||||
level = str(fields.get("seniority") or "")
|
||||
if level and not json.loads(row["raw"] or "{}").get("seniority"):
|
||||
sets["seniority"] = level
|
||||
if level:
|
||||
sets["seniority_model"] = sets["seniority"] = level
|
||||
for name in _RAW_INT:
|
||||
number = _int_or_none(fields.get(name))
|
||||
if number is not None and row[name] is None:
|
||||
@@ -1112,10 +1279,12 @@ class Store:
|
||||
languages = [str(code) for code in fields.get("languages") or []]
|
||||
if languages and json.loads(row["languages"] or "[]") == []:
|
||||
sets["languages"] = json.dumps(languages)
|
||||
filled = len(sets)
|
||||
filled = len(sets) - ("seniority" in sets)
|
||||
if "place" in sets:
|
||||
sets["place_key"] = place_key(sets["place"])
|
||||
sets["place_fold"] = places.fold(sets["place"])
|
||||
gazetteer.write(conn, self._gazetteer(conn), uid,
|
||||
gazetteer.texts({"place": sets["place"]}))
|
||||
sets["extracted_at"] = self.now()
|
||||
conn.execute(
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
||||
@@ -1375,8 +1544,9 @@ class Store:
|
||||
as a column so a gated role keeps its row and reason."""
|
||||
columns: list[str] = []
|
||||
params: list[Any] = []
|
||||
gaz = self.gazetteer()
|
||||
for n, gate in enumerate(gates):
|
||||
empty, match, values = _gate_sql(gate)
|
||||
empty, match, values = _gate_sql(gate, gaz)
|
||||
columns.append(
|
||||
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
||||
params += values
|
||||
|
||||
Reference in new issue
Block a user