Places as a gazetteer, role_places, teaser and three seniority sources

Migration 19 adds places, place_aliases, place_includes (seeded from the
dictionaries in app/models.py and app/places.py, Europe as a continent over
the European countries) and role_places, one row per place of a role,
resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them
from raw["places"], else the place line cut at ";"; the migration builds them
from each role's place column.

The Place filter and the country and place gates are joins over the tree:
country:DK is any place under Denmark, region:skane any town in Skane, a
country gate is met by any place in its countries, and the EU fallback of
the word matcher is no longer used. A place: token the gazetteer does not
know still matches whole words. Store gains gazetteer, role_places,
unresolved_places, add_place_alias, add_place and relink_places.

roles.teaser is kept-filled and embedded while there is no description; a
vector is dropped when a better text arrives. seniority_model, _stated and
_title sit behind roles.seniority, which stays the first filled of them.

TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads
READS.fields.posted and falls back to the old set for undeclared readers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:31:45 +02:00
1 parent 4c0da58da7
commit bbd7522163
10 files changed
+1437 -98

No files matched your search

+225 -55
View File
@@ -22,14 +22,14 @@ import numpy as np
from desk.models import Item
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
ScoredRole,
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
)
from app.seniority import from_title
from app.storage import schema
from app.storage import gazetteer, schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
@@ -101,7 +101,8 @@ _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
#: one, else the description's opening. A `SUBSTR` of the source column, not
#: the column itself, so a big `description` never leaves SQLite whole for a
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
_EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), "
"roles.teaser)")
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
@@ -177,23 +178,34 @@ def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
#: A role with no place the gazetteer resolved, over `roles r`.
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
"""One gate as `(empty, match, params)`: SQL that is true when the role's
column is empty, and SQL that is true when it matches, over `roles r`.
`params` belong to `match`; `empty` takes none."""
`params` belong to `match`; `empty` takes none.
`country` and `place` match over the gazetteer, as the feed's Place
filter does (`_places_sql`): a country gate is met by a role whose
`country` is one of its codes or that has a place in one of them, so
`DK` is met by `Copenhagen; London`. Its column is empty only when the
role has neither. A place gate's value is a place token, bare text a
`place:`."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "country":
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
codes=[v.upper() for v in values])
return f"(r.country = '' AND {_UNPLACED})", match, params
if gate.field == "place":
# Whole folded words, as the feed's Place filter: `lund` is not in
# `Billund`. A value is a place token, bare text a `place:`.
clauses = [places.match_sql(v, place="r.place_fold", country="r.country")
for v in values]
match = " OR ".join(sql for sql, _ in clauses) or "0"
return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps]
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
@@ -205,18 +217,54 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]:
"""The Place filter over `roles`: any of the tokens, each a `place_match`
call (`places.match_sql`). ValueError for a token that names nothing,
which would otherwise match no role without saying why."""
clauses, params = [], []
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
"""Any of the place tokens over `table`, as a join over the gazetteer: a
role with a place under one of the places the tokens name (`country:DK`
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
Malmö by any spelling), or whose `country` is one of their countries (a
role whose board gave a country and no place). `codes` are more values of
`country` to accept. A `place:` token the gazetteer does not know is
matched on whole words of the role's folded line (`places.match_sql`),
so a town nobody has entered yet is still found."""
ids: set[int] = set()
found = {c for c in codes if c}
clauses: list[str] = []
params: list[Any] = []
for token in tokens:
named = gaz.lookup(token)
if named is None:
sql, values = places.match_sql(token, place=f"{table}.place_fold",
country=f"{table}.country")
clauses.append(sql)
params += values
continue
under = gaz.under(named)
ids |= under
found |= gaz.codes(under)
if ids:
clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places "
"WHERE place_id IN (SELECT value FROM json_each(?)))")
params.insert(0, json.dumps(sorted(ids)))
if found:
clauses.insert(1 if ids else 0,
f"{table}.country IN (SELECT value FROM json_each(?))")
params.insert(1 if ids else 0, json.dumps(sorted(found)))
return f"({' OR '.join(clauses) or '0'})", params
def _known_places(tokens: Sequence[str]) -> None:
"""ValueError for a token that names nothing, which would otherwise match
no role without saying why."""
for token in tokens:
if not places.known(token):
raise ValueError(f"{token!r} is not a place this app knows")
sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country")
clauses.append(sql)
params += values
return f"({' OR '.join(clauses)})", params
def _text_rank(description: str, teaser: str) -> int:
"""What a role is embedded from, as a rank: 2 its description, 1 its
teaser, 0 its title alone."""
return 2 if description else 1 if teaser else 0
def _int_or_none(value: Any) -> int | None:
@@ -351,6 +399,8 @@ class Store:
self.path.parent.mkdir(parents=True, exist_ok=True)
#: What "now" is. Replaced by tests that need two writes a second apart.
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
#: The gazetteer as last read, with the `gazetteer.VERSION` it was read at.
self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
@@ -364,7 +414,7 @@ class Store:
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
registered on every one, since the place filter and the place gate
call it (`places.match_sql`)."""
call it for a town the gazetteer does not know (`_places_sql`)."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
places.register(conn)
@@ -400,6 +450,103 @@ class Store:
def now(self) -> str:
return _iso(self.clock())
# ------------------------------------------------------------- gazetteer
def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer:
"""The gazetteer, read again only when its tables have grown, which
another process may have done: one cheap query a call."""
version = tuple(conn.execute(gazetteer.VERSION).fetchone())
if self._gaz is None or self._gaz[0] != version:
self._gaz = (version, gazetteer.load(conn))
return self._gaz[1]
def gazetteer(self) -> places.Gazetteer:
with closing(self._connect()) as conn:
return self._gazetteer(conn)
def role_places(self, uid: str) -> list[RolePlace]:
"""One role's places in the host's order, a copy's after the role's own."""
return [RolePlace.from_row(r) for r in self._rows(
"""SELECT rp.position, rp.text, rp.place_id, rp.url,
COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind
FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id
WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""",
(uid,))]
def unresolved_places(self, open_only: bool = True,
limit: int | None = None) -> list[tuple[str, str, int]]:
"""`(fold, text, roles)` for every place text no place of the
gazetteer resolves, most roles first: the list a person works through
with `add_place_alias` and `add_place`. `text` is one spelling of the
fold. A text that names no place ("Remote", `places.placeless`) is
left out; with `open_only`, so are closed roles and copies."""
where = "rp.place_id IS NULL"
if open_only:
where += " AND r.closed_at IS NULL AND r.dup_of IS NULL"
rows = self._rows(
f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n
FROM role_places rp JOIN roles r ON r.uid = rp.uid
WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""")
found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows
if not places.placeless(r["fold"])]
return found[:limit] if limit is not None else found
def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int:
"""A place by id, or by a token that names exactly one place."""
if isinstance(place, int):
if place not in gaz.places:
raise ValueError(f"there is no place {place}")
return place
named = gaz.lookup(place) or frozenset()
if len(named) != 1:
raise ValueError(f"{place!r} names {len(named)} places, not one")
return next(iter(named))
def add_place_alias(self, text: str, place: int | str) -> int:
"""Write `text` as a spelling of `place` (an id, or a token such as
`place:copenhagen` or `country:DK` naming one place), and resolve again
every role with a place text that holds it. Returns how many roles now
resolve differently."""
folded = places.fold(text)
if not folded:
raise ValueError("an alias needs a letter or a digit")
with self.transaction() as conn:
place_id = self._place_id(self._gazetteer(conn), place)
conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
(folded, place_id))
return gazetteer.relink(conn, self._gazetteer(conn), folded)
def add_place(self, name: str, kind: str, parent: int | str,
aliases: Sequence[str] = ()) -> int:
"""Add a city or a region the gazetteer lacks, in `parent` (an id or a
token), written as `name` and each of `aliases`, and resolve again the
roles with a place text that holds one of them. Returns its id."""
if kind not in ("city", "region"):
raise ValueError(f"a place added by hand is a city or a region, not {kind!r}")
folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f))
if not folds:
raise ValueError("a place needs a name")
with self.transaction() as conn:
parent_id = self._place_id(self._gazetteer(conn), parent)
if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?",
(kind, folds[0])).fetchone():
raise ValueError(f"there is already a {kind} {name!r}")
place_id = int(conn.execute(
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
(kind, folds[0], name.strip(), parent_id)).lastrowid or 0)
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
[(f, place_id) for f in folds])
gaz = self._gazetteer(conn)
for folded in folds:
gazetteer.relink(conn, gaz, folded)
return place_id
def relink_places(self) -> int:
"""Resolve every role's own places again, after the gazetteer was
seeded anew. Returns how many roles now resolve differently."""
with self.transaction() as conn:
return gazetteer.relink(conn, self._gazetteer(conn))
# ---------------------------------------------------------------- roles
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
@@ -426,19 +573,32 @@ class Store:
alone for good, whatever the plugin's raw guess now says — see
`set_extracted` for the other half of the rule.
`seniority` is the title's level (`seniority.from_title`) until
extract has read the role and extract's from then on, as `function`
is. The classifier is called here rather than in a plugin, so a role
from any source gets it, the way `country` comes from `place` here
too. A level the board states in `raw` is a board field and wins over
both. `place_fold` is written with `place`, under the same keep-filled
rule.
`seniority` is the first filled of three levels. `seniority_model`
is extract's (`set_extracted`) and never written here.
`seniority_stated` is the host's word as `raw["seniority"]` carries
it, already in `SENIORITIES` (anything else is ''), and
`seniority_title` is `seniority.from_title`; both are rewritten every
pull. The classifier is called here rather than in a plugin, so a
role from any source gets it, the way `country` comes from `place`.
The role's places (`role_places`) are the texts of `raw["places"]`,
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
against the gazetteer; `place` is the line shown, the texts joined
when `raw` has none. Like `place`, a pull with no place leaves them
as they are. `place_fold` is written with `place`, under the same
keep-filled rule. `teaser` is kept-filled too.
A role is embedded from its description, else its teaser, else its
title (`app.jobs.embed.document`). When a pull brings a better one of
these than the role had, its vector is dropped so the next `embed`
reads it.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
raw = item.raw or {}
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
place_texts = gazetteer.texts(raw)
place = str(raw.get("place") or "") or "; ".join(place_texts)
stamp = self.now()
published = _iso(item.published_at)
row: dict[str, Any] = {
@@ -463,12 +623,16 @@ class Store:
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row["place"] = place
row["country"] = row["country"] or country_of(place)
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
stated = str(raw.get("seniority") or "")
row["seniority"] = stated or from_title(item.title)
row["seniority_stated"] = stated if stated in SENIORITIES else ""
row["seniority_title"] = from_title(item.title)
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
row["teaser"] = str(raw.get("teaser") or "")
row["place_fold"] = places.fold(row["place"])
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
@@ -476,13 +640,13 @@ class Store:
row["salary_text"] = str(raw.get("salary_text") or "")
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
"feed_text", "original_url", "teaser", *_RAW_TEXT)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text")]
"salary_text", "seniority_stated", "seniority_title")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -493,27 +657,29 @@ class Store:
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
"ELSE roles.function END")
# The title's level follows the title until extract has read the
# role, and is extract's from then on; a board's stated level wins.
updates.append("seniority=excluded.seniority" if stated else
"seniority=CASE WHEN roles.extracted_at IS NULL "
"THEN excluded.seniority ELSE roles.seniority END")
# The model's level, else the host's, else the title's.
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
updates.append("place_fold=CASE WHEN excluded.place != '' "
"THEN excluded.place_fold ELSE roles.place_fold END")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
with self.transaction() as conn:
before = conn.execute("SELECT description FROM roles WHERE uid=?",
before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?",
(row["uid"],)).fetchone()
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
# A role listed without its text is embedded from its title, and
# `embed` encodes a role once. When the text arrives on a later
# pull, as it does for a host that states it on the role's own
# page, the vector goes, so the next `embed` reads the text.
if before is not None and not before["description"] and row["description"]:
if place_texts:
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
# `embed` encodes a role once, from the best text it has. When a
# later pull brings a better one, as a host that states the text
# on the role's own page does, the vector goes, so the next
# `embed` reads it.
if before is not None and _text_rank(row["description"] or before["description"],
row["teaser"] or before["teaser"]) \
> _text_rank(before["description"], before["teaser"]):
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
return before is None
@@ -533,6 +699,7 @@ class Store:
role = ScoredRole.from_row(row)
role.tags = self._tags_of(uid)
role.copies = self._copies_of(uid)
role.places = self.role_places(uid)
return role
def _copies_of(self, uid: str) -> list[RoleCopy]:
@@ -685,7 +852,7 @@ class Store:
`company_keys` keeps those exact companies. `places` keeps a role any
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
`place:malmo`), matched on whole folded words; a token that names
`place:malmo`), over the gazetteer (`_places_sql`); a token that names
nothing is a ValueError. `seniority` keeps the roles at any of those
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
with no level, which is most of them. An empty list is no filter.
@@ -934,8 +1101,8 @@ class Store:
else:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
if places:
# Last, so the cheaper terms have cut the rows it is called on.
clause, values = _places_sql(places)
_known_places(places)
clause, values = _places_sql(places, self.gazetteer())
where.append(clause)
params += values
sql = f"FROM roles {_WITH_VOTES}"
@@ -1087,9 +1254,9 @@ class Store:
non-empty answer overwrites it outright rather than only filling an
empty column — see `upsert_role`'s docstring for the other half of
the rule that then keeps a later pull from putting the guess back.
`seniority` is the other: `upsert_role` fills it from the title, so a
non-empty answer replaces it, unless the board stated a level in
`raw`."""
`seniority` is the other: a non-empty answer is `seniority_model`,
which comes before the stated and the title's level, so it becomes
`seniority` too."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1103,8 +1270,8 @@ class Store:
if function:
sets["function"] = function
level = str(fields.get("seniority") or "")
if level and not json.loads(row["raw"] or "{}").get("seniority"):
sets["seniority"] = level
if level:
sets["seniority_model"] = sets["seniority"] = level
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
@@ -1112,10 +1279,12 @@ class Store:
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets)
filled = len(sets) - ("seniority" in sets)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
gazetteer.write(conn, self._gazetteer(conn), uid,
gazetteer.texts({"place": sets["place"]}))
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
@@ -1375,8 +1544,9 @@ class Store:
as a column so a gated role keeps its row and reason."""
columns: list[str] = []
params: list[Any] = []
gaz = self.gazetteer()
for n, gate in enumerate(gates):
empty, match, values = _gate_sql(gate)
empty, match, values = _gate_sql(gate, gaz)
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values