Places as a gazetteer, role_places, teaser and three seniority sources

Migration 19 adds places, place_aliases, place_includes (seeded from the
dictionaries in app/models.py and app/places.py, Europe as a continent over
the European countries) and role_places, one row per place of a role,
resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them
from raw["places"], else the place line cut at ";"; the migration builds them
from each role's place column.

The Place filter and the country and place gates are joins over the tree:
country:DK is any place under Denmark, region:skane any town in Skane, a
country gate is met by any place in its countries, and the EU fallback of
the word matcher is no longer used. A place: token the gazetteer does not
know still matches whole words. Store gains gazetteer, role_places,
unresolved_places, add_place_alias, add_place and relink_places.

roles.teaser is kept-filled and embedded while there is no description; a
vector is dropped when a better text arrives. seniority_model, _stated and
_title sit behind roles.seniority, which stays the first filled of them.

TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads
READS.fields.posted and falls back to the old set for undeclared readers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:31:45 +02:00
1 parent 4c0da58da7
commit bbd7522163
10 files changed
+1437 -98

No files matched your search

+6 -2
View File
@@ -2,7 +2,10 @@
The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors`
(see `Store.put_vectors`). What gets encoded is the title and the opening of
the description, which is what says what a role is.
the description, which is what says what a role is; while a role has no
description, the teaser its host's list carried, and while it has neither,
the title alone. `Store.upsert_role` drops the vector when a better text
arrives, so the role is encoded again.
sentence-transformers is optional, as in newsdesk. Without it this task puts
one notice on screen and does nothing; `score` then leaves `fit` out and ranks
@@ -55,7 +58,8 @@ def wants_e5_prefix(model: str) -> bool:
def document(role: Role) -> str:
return f"{role.title}\n{role.description[:BODY_CHARS]}"
"""The title and the description, else the teaser."""
return f"{role.title}\n{(role.description or role.teaser)[:BODY_CHARS]}"
def role_text(model: str, role: Role) -> str:
+73 -16
View File
@@ -12,7 +12,10 @@ from __future__ import annotations
import json
import re
import sys
import unicodedata
from functools import lru_cache
from pathlib import Path
from typing import Annotated, Any, Literal, get_args
from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator
@@ -358,8 +361,9 @@ _REGIONS = ("europe", "emea", "eu", "european union", "eea")
#: state or a Canadian province in the same part of the line beats one of
#: these ("London, Ontario" is Canada); a country name does not lose to it.
_CITIES: dict[str, str] = {
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
**dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck"), "AT"),
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege",
"luik"), "BE"),
**dict.fromkeys(("sofia",), "BG"),
**dict.fromkeys(("zagreb",), "HR"),
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
@@ -369,7 +373,7 @@ _CITIES: dict[str, str] = {
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
**dict.fromkeys(("tallinn", "tartu"), "EE"),
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
**dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"),
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
"lille", "rennes", "amiens"), "FR"),
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
@@ -378,27 +382,30 @@ _CITIES: dict[str, str] = {
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck"), "DE"),
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
**dict.fromkeys(("budapest",), "HU"),
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
"florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples",
"napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"),
**dict.fromkeys(("riga",), "LV"),
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
**dict.fromkeys(("luxembourg", "luxemburg"), "LU"),
**dict.fromkeys(("valletta",), "MT"),
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
"delft"), "NL"),
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
"PL"),
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"),
**dict.fromkeys(("belgrade", "beograd"), "RS"),
**dict.fromkeys(("bratislava",), "SK"),
**dict.fromkeys(("ljubljana",), "SI"),
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
"granada", "santiago de compostela"), "ES"),
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras",
"linkoping", "orebro", "norrkoping", "jonkoping"), "SE"),
**dict.fromkeys(("reykjavik",), "IS"),
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
@@ -408,7 +415,7 @@ _CITIES: dict[str, str] = {
"slough", "southampton", "swindon", "liverpool", "redhill",
"milton keynes"), "GB"),
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
"sankt gallen", "st gallen"), "CH"),
"sankt gallen", "st gallen", "lucerne", "luzern"), "CH"),
**dict.fromkeys(("kyiv", "kiev"), "UA"),
**dict.fromkeys(("istanbul", "ankara"), "TR"),
**dict.fromkeys(("tirana",), "AL"),
@@ -659,9 +666,10 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
# -------------------------------------------------------------- posting dates
#: Hosts whose date is a creation or first-publication timestamp, safe to
#: read as when a role was posted, checked against live boards on
#: 2026-09-29:
#: Hosts whose date is a creation or first-publication timestamp, for a
#: reader that does not declare `READS` yet (see `trusted_posted`); a reader
#: that declares it says so with `fields.posted="trusted"`. Checked against
#: live boards on 2026-09-29:
#:
#: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers
#: `first_published` over `updated_at`. A live board (adyen) showed a job
@@ -708,14 +716,39 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
#: HR Manager) state a date nobody checked, and Pinpoint states none.
#:
#: A host not listed here, or a role whose stored `published_at` fails the
#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the
#: safe default this migration is replacing everywhere else.
TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({
#: sanity check in `posted_at`, keeps `first_seen` as its posted date.
_UNDECLARED_TRUSTED: frozenset[str] = frozenset({
"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
"personio", "teamtailor", "jobindex", "adzuna", "platsbanken",
"workday", "successfactors", "oracle", "phenom", "bamboohr", "jobylon", "rippling",
})
@lru_cache(maxsize=1)
def _declared_posted() -> dict[str, str]:
"""`fields.posted` of every reader that declares `READS`, by name. Read
once: the readers are loaded when `plugins.readers` is imported. `plugins`
is a top-level folder beside `app`, reached as `app/jobs/extract.py`
reaches it."""
root = str(Path(__file__).resolve().parents[1])
if root not in sys.path:
sys.path.insert(0, root)
from plugins.readers import READERS
return {name: module.READS.fields.posted for name, module in READERS.items()
if getattr(module, "READS", None) is not None}
def trusted_posted(host: str) -> bool:
"""Whether `host` states a date that is safe to read as when a role was
posted: its reader declares `fields.posted="trusted"`, or, for a reader
without `READS` (or a search provider not yet a reader), it is in
`_UNDECLARED_TRUSTED`."""
declared = _declared_posted()
if host in declared:
return declared[host] == "trusted"
return host in _UNDECLARED_TRUSTED
#: A `published_at` before this is a parsing slip or a placeholder, not a
#: real posting date: no board here existed before 2015.
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
@@ -732,7 +765,7 @@ def posted_host(raw: dict[str, Any]) -> str:
def posted_at(published_at: str, first_seen: str, host: str) -> str:
"""The date a role counts as posted: the earlier of `first_seen` (when
the app itself first stored it) and `published_at` (what the board
claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and
claims), but only when `trusted_posted(host)` and
`published_at` is neither absurdly old nor later than `first_seen` — a
role cannot be posted after the app already saw it, trusted host or not,
which also catches a `published_at` the board reports in the future
@@ -740,7 +773,7 @@ def posted_at(published_at: str, first_seen: str, host: str) -> str:
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
is exact and needs no parsing. A role with no trusted date, or with one
that fails the check, keeps `first_seen`."""
if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen:
if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen:
return published_at
return first_seen
@@ -775,6 +808,20 @@ class RoleCopy(Model):
posted_at: str = ""
class RolePlace(Model):
"""One place of a role (`role_places`): the host's `text`, and the place
of the gazetteer it resolves to, `place_id` None (and `name` and `kind`
empty) while it resolves to none. `url` is the link of a copy folded into
the role, '' for the role's own places."""
position: int = 0
text: str = ""
place_id: int | None = None
name: str = ""
kind: str = ""
url: str = ""
class Role(Model):
"""A row of `roles`: one posting on one board."""
@@ -797,13 +844,20 @@ class Role(Model):
#: could be normalised into `salary_min`/`salary_max`/`currency`
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
salary_text: str = ""
#: The level shown and filtered on: the first filled of `seniority_model`,
#: `seniority_stated` and `seniority_title`.
seniority: str = ""
seniority_model: str = ""
seniority_stated: str = ""
seniority_title: str = ""
function: str = ""
permit: Permit = ""
#: ISO 639-1 codes the posting says are required, lowercase.
languages: JsonStrings = []
description: str = ""
feed_text: str = ""
#: The short text the host's list carries, until `description` arrives.
teaser: str = ""
original_url: str = ""
published_at: str
#: The date this role counts as posted: see `posted_at`. Used for
@@ -838,6 +892,9 @@ class Role(Model):
#: The open copies of this role, oldest first, when it is the kept one;
#: empty otherwise. Not a column: filled by `Store.role`.
copies: list[RoleCopy] = []
#: Every place of the role, in the host's order. Not a column: filled by
#: `Store.role` from `role_places`.
places: list[RolePlace] = []
@computed_field # type: ignore[prop-decorator]
@property
+313 -5
View File
@@ -19,8 +19,13 @@ several hundred milliseconds a query, as a set intersection on the words of a
line it costs a few. `register` gives a connection that answer as the SQL
function `place_match`, and `match_sql` returns the clause that calls it.
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
chip's text back from the address. This module opens no database; the caller
hands over the places it holds, or its connection.
chip's text back from the address.
The filter and the gates no longer match words: a role's places are resolved
against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
and a token is the places under the place it names. `matches` stays the rule
for a `place:` token the gazetteer does not know. This module opens no
database; the caller hands over the places it holds, or its connection.
"""
from __future__ import annotations
@@ -32,10 +37,12 @@ from collections.abc import Iterable
from dataclasses import dataclass
from functools import lru_cache
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
country_of)
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed",
"Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse",
"placeless", "register", "seed", "suggest", "towns"]
def fold(text: str) -> str:
@@ -538,6 +545,9 @@ _PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Danish boards append "and the possibility of working from home".
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
#: A slash between two places, "Malmö/Lund", "Berlin / Munich".
_SLASH = re.compile(r"\s*/\s*")
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
@@ -665,3 +675,301 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
regions.sort(key=lambda s: (-s.count, s.label))
towns.sort(key=lambda s: (-s.count, s.label))
return (countries + regions + towns)[:limit]
# ------------------------------------------------------------------ gazetteer
#: The kinds of place, widest first: the index is how specific a kind is.
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
#: Entries of `_CITIES` that name a region rather than a town.
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
"bay area"))
#: Words a place text holds when it names how or where-ever a role is worked
#: rather than a place: "Remote", "Multiple Locations", "Home based". A text
#: of these alone is no place, so it is never listed as unresolved.
_PLACELESS = frozenset((
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
def placeless(text: str) -> bool:
"""Whether a place text names no place at all: empty, or only numbers and
words such as `Remote`, `Hybrid`, `2 Locations`."""
return all(word in _PLACELESS or word.isdigit() for word in fold(text).split())
@dataclass(frozen=True)
class Seed:
"""One place as the code knows it: `key` is unique within `kind` (an ISO
code for a country, a region's key, a town's group), `parent` the
`(kind, key)` of the place it lies in, `aliases` the folds it is written
as."""
kind: str
key: str
name: str
parent: tuple[str, str] | None
aliases: tuple[str, ...]
def _subdivisions() -> list[tuple[str, str, str]]:
"""`(country, code, name)` of every US state and Canadian province."""
found = []
for line in _SUBDIVISIONS.split("\n"):
if line:
country, rest = line.split(" ", 1)
for entry in rest.split("|"):
name, code = entry.strip().rsplit(" ", 1)
found.append((country, code, name))
return found
def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
"""The gazetteer the dictionaries in code describe, parents before their
children, and the inclusions beyond the tree (Øresund holds the Capital
Region and Skåne, which lie in two countries).
Europe is a continent over the countries of `_EUROPE`, and is also what
`EU`, `EMEA` and `European Union` name. Every country of `_COUNTRY_NAMES`
by its names; the US states and Canadian provinces as regions; the
regions of `REGIONS`; the towns of the alias groups, of the regions and
of `_CITIES`, one place per group (`Copenhagen`, `København`,
`Kopenhamn` are one), in their region when a region lists them and else
in their country."""
europe = ("continent", "europe")
seeds: dict[tuple[str, str], Seed] = {}
def add(kind: str, key: str, name: str, parent: tuple[str, str] | None,
aliases: Iterable[str]) -> None:
folds = tuple(dict.fromkeys(a for a in aliases if a))
if (kind, key) in seeds:
old = seeds[(kind, key)]
folds = tuple(dict.fromkeys((*old.aliases, *folds)))
seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds)
else:
seeds[(kind, key)] = Seed(kind, key, name, parent, folds)
add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS)))
for code, names in _COUNTRY_LABELS.items():
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
for country, code, name in _subdivisions():
add("region", f"{country}-{code}", name, ("country", country), (fold(name),))
for region in REGIONS.values():
parent = ("country", region.country) if region.country else europe
add("region", region.key, region.label, parent,
(v for name in (region.key, region.label, *region.names)
for v in _variants(fold(name))))
for name in sorted(_CITY_REGIONS):
add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,))
# Towns, one per group, in the region that lists them first.
town_region: dict[str, str] = {}
for region in REGIONS.values():
for name in region.towns:
town_region.setdefault(_group(fold(name)), region.key)
folds: dict[str, dict[str, None]] = {}
shown: dict[str, str] = {}
def add_town(folded: str, name: str) -> None:
key = _group(folded)
folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded)))
shown.setdefault(key, name)
for spellings in _ALIAS_GROUPS:
for spelling in spellings:
add_town(fold(spelling), spellings[0])
for region in REGIONS.values():
for name in region.towns:
add_town(fold(name), name)
for city in _CITIES:
if city not in _CITY_REGIONS and city not in _REGION_BY_NAME:
add_town(city, _TOWN_LABELS.get(city) or city.title())
for key, forms in folds.items():
country = next((_CITIES[f] for f in forms if f in _CITIES), "")
if key in town_region:
parent = ("region", town_region[key])
elif country:
parent = ("country", country)
else:
raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES")
add("city", key, shown[key], parent, forms)
includes = [(("region", region.key), ("region", part))
for region in REGIONS.values() for part in region.includes]
return list(seeds.values()), includes
@dataclass(frozen=True)
class Place:
"""A row of `places`."""
id: int
kind: str
key: str
name: str
parent_id: int | None
class Gazetteer:
"""The places, their aliases and what lies in what, as the database holds
them, read into memory: what resolves a role's place text to places and a
filter token to the places under it. Built by the store from its tables,
and rebuilt when they change."""
def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]],
aliases: Iterable[tuple[str, int]],
includes: Iterable[tuple[int, int]] = ()) -> None:
self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows}
self.by_key: dict[tuple[str, str], int] = {
(p.kind, p.key): p.id for p in self.places.values()}
by_fold: dict[str, list[int]] = {}
for folded, place_id in aliases:
by_fold.setdefault(folded, []).append(place_id)
self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()}
self._longest = max((len(f.split()) for f in self.aliases), default=1)
self._up: dict[int, set[int]] = {}
self._down: dict[int, set[int]] = {}
for p in self.places.values():
if p.parent_id is not None:
self._link(p.parent_id, p.id)
for whole, part in includes:
self._link(whole, part)
self._ancestors: dict[int, frozenset[int]] = {}
self._resolved: dict[str, tuple[int, ...]] = {}
def _link(self, whole: int, part: int) -> None:
self._up.setdefault(part, set()).add(whole)
self._down.setdefault(whole, set()).add(part)
def ancestors(self, place_id: int) -> frozenset[int]:
"""Every place `place_id` lies in, through parents and inclusions."""
if (found := self._ancestors.get(place_id)) is None:
seen: set[int] = set()
todo = list(self._up.get(place_id, ()))
while todo:
up = todo.pop()
if up not in seen:
seen.add(up)
todo += self._up.get(up, ())
found = self._ancestors[place_id] = frozenset(seen)
return found
def under(self, ids: Iterable[int]) -> frozenset[int]:
"""The places and every place that lies in one of them."""
seen: set[int] = set()
todo = list(ids)
while todo:
place_id = todo.pop()
if place_id not in seen:
seen.add(place_id)
todo += self._down.get(place_id, ())
return frozenset(seen)
def country(self, place_id: int) -> str:
"""The ISO code of the country a place is in, '' above or outside one."""
place: Place | None = self.places.get(place_id)
while place is not None and place.kind != "country":
place = self.places.get(place.parent_id) if place.parent_id is not None else None
return place.key if place is not None else ""
def codes(self, ids: Iterable[int]) -> frozenset[str]:
"""The ISO codes of the countries among `ids`, and `EU` with Europe:
what a `roles.country` column holds for a role of one of them."""
found = set()
for place_id in ids:
place = self.places[place_id]
if place.kind == "country":
found.add(place.key)
elif (place.kind, place.key) == ("continent", "europe"):
found.add("EU")
return frozenset(found)
def _specific(self, place_id: int) -> int:
return KINDS.index(self.places[place_id].kind)
def _hits(self, folded: str) -> list[tuple[int, ...]]:
words = folded.split()
hits = []
i = 0
while i < len(words):
for n in range(min(self._longest, len(words) - i), 0, -1):
if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None:
hits.append(ids)
i += n
break
else:
i += 1
return hits
def resolve(self, text: str) -> tuple[int, ...]:
"""The places one place text names, most specific first from the left:
`Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund`
is Malmö and Lund, `Paderborn, Germany` is Germany, `Remote` is none.
Every alias the text holds as whole words is a candidate, longest
first. A fold that names several places is the one in the country
`country_of` reads off the text, if one is, and else the most specific
(`Luxembourg` the town before the country). A candidate that another
one lies in is dropped, and so is one outside the one country the text
names, or, naming none, outside the regions it names in one country:
`London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the
Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several
countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country
`country_of` reads (from a state code, `Paris, TX`, or a code in
parentheses, `Remote (DE)`), the text is that country."""
folded = fold(text)
if (found := self._resolved.get(folded)) is None:
parts = _SLASH.split(_AND_HOME.sub("", text or ""))
found = tuple(dict.fromkeys(
p for part in parts for sub in _PART.split(part) for p in self._resolve(sub)))
self._resolved[folded] = found
return found
def _resolve(self, text: str) -> list[int]:
"""`resolve` for one part of a text, cut where `or`, `/` or `;` end one
place."""
code = country_of(text)
country = self.by_key.get(("country", code)) if code and code != "EU" else None
within = self.under((country,)) if country is not None else frozenset()
picked: list[int] = []
for hit in self._hits(fold(text)):
choice = [p for p in hit if p in within] or list(hit)
best = max(choice, key=self._specific)
if best not in picked:
picked.append(best)
# The widest units named hold the rest: one country, else the regions
# of one country. Several countries are a list, not a path.
for kind in ("country", "region"):
named = [p for p in picked if self.places[p].kind == kind]
if len({self.country(p) for p in named}) == 1:
holds = self.under(named)
picked = [p for p in picked if p in holds or self.places[p].kind == "continent"
or any(p in self.ancestors(q) for q in named)]
if named:
break
wider = {a for p in picked for a in self.ancestors(p)}
picked = [p for p in picked if p not in wider]
if country is not None and not any(p in within or p in self.ancestors(country)
for p in picked):
picked = [country]
return picked
def lookup(self, token: str) -> frozenset[int] | None:
"""The places a filter token names, before what lies in them: `country:DK`
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
every place written so (a town, or a region or a country by name).
None when the gazetteer knows no such place."""
tok = parse(token)
if tok.kind == "country":
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
found = {self.by_key[key]} if key in self.by_key else set()
elif tok.kind == "region":
found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \
else set(self.aliases.get(tok.value, ()))
else:
found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())}
return frozenset(found) or None
+2
View File
@@ -3,6 +3,8 @@
schema.py the tables, and the numbered migrations that bring an older
file to the current shape
store.py `Store`, every SQL statement in the application
gazetteer.py the places tables: seeded from the code, read back, and a
role's places written and relinked
Written for roledesk rather than imported from newsdesk: `desk` has no storage
package on purpose, and the connection plumbing here is a deliberate copy.
+121
View File
@@ -0,0 +1,121 @@
"""The gazetteer's tables: seeding them from the code, reading them back, and
writing a role's places.
`places`, `place_aliases` and `place_includes` hold the places
(`app.places.seed` on a new or migrated file, and what a person adds later).
`role_places` holds one row per place of a role: the host's own text, its
fold, and the place it resolves to, NULL for a text the gazetteer does not
know. A text that names several places ("Malmö/Lund") is one row per place,
each with the same `position` and `text`. A row with a `url` is the place of
a copy folded into the role (`app/dupes.py`) and is not the role's own: only
own rows (`url = ''`) are rewritten here.
"""
from __future__ import annotations
import sqlite3
from collections.abc import Iterable, Mapping, Sequence
from typing import Any
from app import places
#: What changes when the gazetteer does: rows are only ever added.
VERSION = ("SELECT (SELECT MAX(rowid) FROM places), (SELECT MAX(rowid) FROM place_aliases), "
"(SELECT COUNT(*) FROM place_includes)")
def seed(conn: sqlite3.Connection) -> int:
"""Insert what `places.seed` describes, leaving rows already there alone.
Returns how many places were added."""
seeds, includes = places.seed()
ids = {(kind, key): place_id for place_id, kind, key in
conn.execute("SELECT id, kind, key FROM places")}
added = 0
for s in seeds:
if (s.kind, s.key) not in ids:
parent = ids[s.parent] if s.parent else None
ids[(s.kind, s.key)] = conn.execute(
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
(s.kind, s.key, s.name, parent)).lastrowid
added += 1
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
[(a, ids[(s.kind, s.key)]) for a in s.aliases])
conn.executemany("INSERT OR IGNORE INTO place_includes (place_id, part_id) VALUES (?, ?)",
[(ids[whole], ids[part]) for whole, part in includes])
return added
def load(conn: sqlite3.Connection) -> places.Gazetteer:
return places.Gazetteer(
[tuple(r) for r in conn.execute("SELECT id, kind, key, name, parent_id FROM places")],
[tuple(r) for r in conn.execute("SELECT fold, place_id FROM place_aliases")],
[tuple(r) for r in conn.execute("SELECT place_id, part_id FROM place_includes")])
def texts(raw: Mapping[str, Any]) -> list[str]:
"""A role's place texts from `Item.raw`: the `text` of each of `places`
(`[{"text": ..., "remote": ...}]`, the reader contract), else `place` split
at `;`, which is how the older readers join several places into one line.
Blank and repeated texts are dropped."""
given = raw.get("places")
if isinstance(given, list) and given:
found = [str((p.get("text") if isinstance(p, Mapping) else p) or "") for p in given]
else:
found = str(raw.get("place") or "").split(";")
return list(dict.fromkeys(t.strip() for t in found if t and t.strip()))
def rows(gaz: places.Gazetteer, uid: str, place_texts: Sequence[str]
) -> list[tuple[str, int, str, str, int | None]]:
"""`(uid, position, text, fold, place_id)` for each place of the texts."""
found: list[tuple[str, int, str, str, int | None]] = []
for position, text in enumerate(place_texts):
folded = places.fold(text)
found += [(uid, position, text, folded, place_id)
for place_id in (gaz.resolve(text) or (None,))]
return found
_INSERT = "INSERT INTO role_places (uid, position, text, fold, place_id) VALUES (?, ?, ?, ?, ?)"
def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
place_texts: Sequence[str]) -> None:
"""Replace one role's own places with these texts, resolved."""
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
"""`write` for many roles in one statement each way. Returns rows written."""
batch = list(by_uid)
conn.executemany("DELETE FROM role_places WHERE uid = ? AND url = ''",
[(uid,) for uid, _ in batch])
new = [row for uid, found in batch for row in rows(gaz, uid, found)]
conn.executemany(_INSERT, new)
return len(new)
def relink(conn: sqlite3.Connection, gaz: places.Gazetteer, words: str | None = None) -> int:
"""Resolve the own places of every role again, or only of the roles with a
place text that holds `words` (a fold) as whole words, after an alias or a
place was added. Returns how many roles now resolve differently."""
sql = "SELECT uid, position, text, place_id FROM role_places WHERE url = ''"
params: list[Any] = []
if words:
sql += " AND ' ' || fold || ' ' LIKE ? ESCAPE '\\'"
escaped = words.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
params.append(f"% {escaped} %")
sql = ("SELECT uid, position, text, place_id FROM role_places WHERE url = '' AND uid IN "
f"(SELECT uid FROM ({sql}))")
before: dict[str, dict[tuple[int, str], set[int | None]]] = {}
for uid, position, text, place_id in conn.execute(sql + " ORDER BY uid, position", params):
before.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
changed = []
for uid, held in before.items():
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in held}
if now != held:
changed.append((uid, [text for _, text in sorted(held)]))
write_many(conn, gaz, changed)
return len(changed)
+156 -7
View File
@@ -18,8 +18,9 @@ import sqlite3
from collections.abc import Callable
from app import places
from app.models import country_of, posted_at, posted_host
from app.models import SENIORITIES, country_of, posted_at, posted_host
from app.seniority import from_title
from app.storage import gazetteer
log = logging.getLogger(__name__)
@@ -44,8 +45,9 @@ CREATE TABLE IF NOT EXISTS roles (
-- require the country: "UK | Remote" is not remote from Berlin.
place TEXT NOT NULL DEFAULT '',
place_key TEXT NOT NULL DEFAULT '',
-- places.fold(place): what the Place filter and the place gate match
-- whole words against, through the place_match function (app/places.py).
-- places.fold(place): what a place: token the gazetteer does not know is
-- matched against as whole words, through place_match (app/places.py).
-- The filter and the gates otherwise read role_places.
place_fold TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
@@ -53,7 +55,8 @@ CREATE TABLE IF NOT EXISTS roles (
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
-- seniority.from_title(title) at ingest, until extract reads the posting.
-- The level shown and filtered on: the first filled of seniority_model,
-- seniority_stated and seniority_title (the last columns), written with them.
seniority TEXT NOT NULL DEFAULT '',
function TEXT NOT NULL DEFAULT '',
-- Whether the posting says it sponsors a work permit, and the languages
@@ -69,7 +72,7 @@ CREATE TABLE IF NOT EXISTS roles (
-- stored the role; posted_at (app.models.posted_at) is the earlier of
-- first_seen and published_at, trusted only for a host known to report a
-- creation date rather than one that moves on an edit
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
-- (app.models.trusted_posted) — recency and the feed's newest order
-- read posted_at, first_seen stays "when the app found it". closed_at is
-- set when a board fetched in full no longer lists the role, and cleared
-- if it returns.
@@ -100,7 +103,62 @@ CREATE TABLE IF NOT EXISTS roles (
-- precedes, which the migration tests do.
dup_of TEXT,
created_at TEXT NOT NULL,
salary_text TEXT NOT NULL DEFAULT ''
salary_text TEXT NOT NULL DEFAULT '',
teaser TEXT NOT NULL DEFAULT '',
seniority_model TEXT NOT NULL DEFAULT '',
seniority_stated TEXT NOT NULL DEFAULT '',
seniority_title TEXT NOT NULL DEFAULT ''
);
-- roles.teaser is the short text a list carries, kept until the full text
-- arrives and embedded while there is none; a later empty teaser does not
-- wipe it. seniority_model, seniority_stated and seniority_title are the
-- three levels behind roles.seniority: the model's (extract), the host's own
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
-- among the last columns: SQLite cannot drop a column a comment precedes,
-- which the migration tests do.
-- The gazetteer: a place is a continent, a country (key the ISO code), a
-- region or a city, in the place parent_id names. key is unique per kind:
-- a region's key (app.places.REGIONS), a town's group. Seeded from the
-- dictionaries in app/models.py and app/places.py (app.places.seed).
CREATE TABLE IF NOT EXISTS places (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
key TEXT NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER REFERENCES places(id),
UNIQUE (kind, key)
);
-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn
-- and copenhagen are Copenhagen. One fold may name several places
-- (Luxembourg the town and the country).
CREATE TABLE IF NOT EXISTS place_aliases (
fold TEXT NOT NULL,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
UNIQUE (fold, place_id)
);
-- What a place holds beyond its children: Øresund holds the Capital Region
-- of Denmark and Skåne, which lie in two countries.
CREATE TABLE IF NOT EXISTS place_includes (
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (place_id, part_id)
);
-- One row per place of a role (app/storage/gazetteer.py): the host's text in
-- the host's order (position), its fold, and the place it resolves to, NULL
-- while the gazetteer does not know it. A text naming several places is a
-- row for each. url is the link of a copy folded into this role, '' for
-- the role's own places.
CREATE TABLE IF NOT EXISTS role_places (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
position INTEGER NOT NULL,
text TEXT NOT NULL,
fold TEXT NOT NULL,
place_id INTEGER REFERENCES places(id),
url TEXT NOT NULL DEFAULT ''
);
-- The open vocabulary summarise reads off a posting: the role family, the
@@ -303,6 +361,11 @@ CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
-- Partial: all but about 3% of roles are no copy.
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
ON roles(dup_of) WHERE dup_of IS NOT NULL;
-- A role's places, and the roles in a set of places (the Place filter and
-- the country and place gates, `uid IN (... WHERE place_id IN ...)`).
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -469,6 +532,10 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
# A Python step: the first rebuild runs the rule in app/dupes.py.
(18, "roles.dup_of: one posting listed several times is one role, the others "
"copies of it", lambda conn: _add_dup_of(conn)),
# A Python step: the seed and the resolver are Python.
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
"places, resolved against a gazetteer; roles.teaser; the three seniority "
"sources", lambda conn: _add_places(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -592,6 +659,81 @@ def _add_dup_of(conn: sqlite3.Connection) -> None:
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
_V19 = """
ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT '';
CREATE TABLE places (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
key TEXT NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER REFERENCES places(id),
UNIQUE (kind, key)
);
CREATE TABLE place_aliases (
fold TEXT NOT NULL,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
UNIQUE (fold, place_id)
);
CREATE TABLE place_includes (
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (place_id, part_id)
);
CREATE TABLE role_places (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
position INTEGER NOT NULL,
text TEXT NOT NULL,
fold TEXT NOT NULL,
place_id INTEGER REFERENCES places(id),
url TEXT NOT NULL DEFAULT ''
);
"""
def _add_places(conn: sqlite3.Connection) -> None:
"""Migration 19's step. The tables and columns, the gazetteer seeded, and
every role's places: the `places` of its `raw` where a reader stated
them, else its `place` column cut at `;` (not `raw`'s, which a pull that
read the role's own page sends empty), each resolved once per distinct
text.
The level splits into its sources without changing: on a role extract
has read, `seniority` is the model's; on any other, it is the stated
level when `raw` states one in the vocabulary, else the title's."""
for statement in _statements(_V19):
conn.execute(statement)
gazetteer.seed(conn)
gaz = gazetteer.load(conn)
rows = conn.execute(
"SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall()
by_uid: list[tuple[str, list[str]]] = []
levels = []
for uid, raw, place, title, level, extracted_at in rows:
try:
given = json.loads(raw or "{}")
except ValueError:
given = {}
given = given if isinstance(given, dict) else {}
if found := gazetteer.texts({"places": given.get("places"), "place": place}):
by_uid.append((uid, found))
stated = str(given.get("seniority") or "")
stated = stated if stated in SENIORITIES else ""
if extracted_at is not None:
levels.append((level, stated, from_title(title) if level else "", uid))
elif stated:
levels.append(("", stated, from_title(title), uid))
elif level:
levels.append(("", "", level, uid))
written = gazetteer.write_many(conn, gaz, by_uid)
conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, "
"seniority_title = ? WHERE uid = ?", levels)
log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d",
len(gaz.places), written, len(by_uid), len(levels))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
@@ -659,6 +801,13 @@ def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
conn.execute("COMMIT")
def _create(conn: sqlite3.Connection) -> None:
"""A new file: `SCHEMA`, and the gazetteer seeded."""
for statement in _statements(SCHEMA):
conn.execute(statement)
gazetteer.seed(conn)
def initialise(conn: sqlite3.Connection) -> list[int]:
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
fresh = conn.execute(
@@ -668,7 +817,7 @@ def initialise(conn: sqlite3.Connection) -> list[int]:
if fresh:
# Atomic like a migration: a file half created would otherwise carry
# the marker table and be taken for one to migrate.
_apply(conn, LATEST, SCHEMA)
_apply(conn, LATEST, _create)
else:
applied = migrate(conn)
for statement in _statements(INDEXES):
+225 -55
View File
@@ -22,14 +22,14 @@ import numpy as np
from desk.models import Item
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
ScoredRole,
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
)
from app.seniority import from_title
from app.storage import schema
from app.storage import gazetteer, schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
@@ -101,7 +101,8 @@ _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
#: one, else the description's opening. A `SUBSTR` of the source column, not
#: the column itself, so a big `description` never leaves SQLite whole for a
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
_EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), "
"roles.teaser)")
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
@@ -177,23 +178,34 @@ def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
#: A role with no place the gazetteer resolved, over `roles r`.
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
"""One gate as `(empty, match, params)`: SQL that is true when the role's
column is empty, and SQL that is true when it matches, over `roles r`.
`params` belong to `match`; `empty` takes none."""
`params` belong to `match`; `empty` takes none.
`country` and `place` match over the gazetteer, as the feed's Place
filter does (`_places_sql`): a country gate is met by a role whose
`country` is one of its codes or that has a place in one of them, so
`DK` is met by `Copenhagen; London`. Its column is empty only when the
role has neither. A place gate's value is a place token, bare text a
`place:`."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "country":
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
codes=[v.upper() for v in values])
return f"(r.country = '' AND {_UNPLACED})", match, params
if gate.field == "place":
# Whole folded words, as the feed's Place filter: `lund` is not in
# `Billund`. A value is a place token, bare text a `place:`.
clauses = [places.match_sql(v, place="r.place_fold", country="r.country")
for v in values]
match = " OR ".join(sql for sql, _ in clauses) or "0"
return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps]
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
@@ -205,18 +217,54 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]:
"""The Place filter over `roles`: any of the tokens, each a `place_match`
call (`places.match_sql`). ValueError for a token that names nothing,
which would otherwise match no role without saying why."""
clauses, params = [], []
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
"""Any of the place tokens over `table`, as a join over the gazetteer: a
role with a place under one of the places the tokens name (`country:DK`
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
Malmö by any spelling), or whose `country` is one of their countries (a
role whose board gave a country and no place). `codes` are more values of
`country` to accept. A `place:` token the gazetteer does not know is
matched on whole words of the role's folded line (`places.match_sql`),
so a town nobody has entered yet is still found."""
ids: set[int] = set()
found = {c for c in codes if c}
clauses: list[str] = []
params: list[Any] = []
for token in tokens:
named = gaz.lookup(token)
if named is None:
sql, values = places.match_sql(token, place=f"{table}.place_fold",
country=f"{table}.country")
clauses.append(sql)
params += values
continue
under = gaz.under(named)
ids |= under
found |= gaz.codes(under)
if ids:
clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places "
"WHERE place_id IN (SELECT value FROM json_each(?)))")
params.insert(0, json.dumps(sorted(ids)))
if found:
clauses.insert(1 if ids else 0,
f"{table}.country IN (SELECT value FROM json_each(?))")
params.insert(1 if ids else 0, json.dumps(sorted(found)))
return f"({' OR '.join(clauses) or '0'})", params
def _known_places(tokens: Sequence[str]) -> None:
"""ValueError for a token that names nothing, which would otherwise match
no role without saying why."""
for token in tokens:
if not places.known(token):
raise ValueError(f"{token!r} is not a place this app knows")
sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country")
clauses.append(sql)
params += values
return f"({' OR '.join(clauses)})", params
def _text_rank(description: str, teaser: str) -> int:
"""What a role is embedded from, as a rank: 2 its description, 1 its
teaser, 0 its title alone."""
return 2 if description else 1 if teaser else 0
def _int_or_none(value: Any) -> int | None:
@@ -351,6 +399,8 @@ class Store:
self.path.parent.mkdir(parents=True, exist_ok=True)
#: What "now" is. Replaced by tests that need two writes a second apart.
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
#: The gazetteer as last read, with the `gazetteer.VERSION` it was read at.
self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
@@ -364,7 +414,7 @@ class Store:
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
registered on every one, since the place filter and the place gate
call it (`places.match_sql`)."""
call it for a town the gazetteer does not know (`_places_sql`)."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
places.register(conn)
@@ -400,6 +450,103 @@ class Store:
def now(self) -> str:
return _iso(self.clock())
# ------------------------------------------------------------- gazetteer
def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer:
"""The gazetteer, read again only when its tables have grown, which
another process may have done: one cheap query a call."""
version = tuple(conn.execute(gazetteer.VERSION).fetchone())
if self._gaz is None or self._gaz[0] != version:
self._gaz = (version, gazetteer.load(conn))
return self._gaz[1]
def gazetteer(self) -> places.Gazetteer:
with closing(self._connect()) as conn:
return self._gazetteer(conn)
def role_places(self, uid: str) -> list[RolePlace]:
"""One role's places in the host's order, a copy's after the role's own."""
return [RolePlace.from_row(r) for r in self._rows(
"""SELECT rp.position, rp.text, rp.place_id, rp.url,
COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind
FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id
WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""",
(uid,))]
def unresolved_places(self, open_only: bool = True,
limit: int | None = None) -> list[tuple[str, str, int]]:
"""`(fold, text, roles)` for every place text no place of the
gazetteer resolves, most roles first: the list a person works through
with `add_place_alias` and `add_place`. `text` is one spelling of the
fold. A text that names no place ("Remote", `places.placeless`) is
left out; with `open_only`, so are closed roles and copies."""
where = "rp.place_id IS NULL"
if open_only:
where += " AND r.closed_at IS NULL AND r.dup_of IS NULL"
rows = self._rows(
f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n
FROM role_places rp JOIN roles r ON r.uid = rp.uid
WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""")
found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows
if not places.placeless(r["fold"])]
return found[:limit] if limit is not None else found
def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int:
"""A place by id, or by a token that names exactly one place."""
if isinstance(place, int):
if place not in gaz.places:
raise ValueError(f"there is no place {place}")
return place
named = gaz.lookup(place) or frozenset()
if len(named) != 1:
raise ValueError(f"{place!r} names {len(named)} places, not one")
return next(iter(named))
def add_place_alias(self, text: str, place: int | str) -> int:
"""Write `text` as a spelling of `place` (an id, or a token such as
`place:copenhagen` or `country:DK` naming one place), and resolve again
every role with a place text that holds it. Returns how many roles now
resolve differently."""
folded = places.fold(text)
if not folded:
raise ValueError("an alias needs a letter or a digit")
with self.transaction() as conn:
place_id = self._place_id(self._gazetteer(conn), place)
conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
(folded, place_id))
return gazetteer.relink(conn, self._gazetteer(conn), folded)
def add_place(self, name: str, kind: str, parent: int | str,
aliases: Sequence[str] = ()) -> int:
"""Add a city or a region the gazetteer lacks, in `parent` (an id or a
token), written as `name` and each of `aliases`, and resolve again the
roles with a place text that holds one of them. Returns its id."""
if kind not in ("city", "region"):
raise ValueError(f"a place added by hand is a city or a region, not {kind!r}")
folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f))
if not folds:
raise ValueError("a place needs a name")
with self.transaction() as conn:
parent_id = self._place_id(self._gazetteer(conn), parent)
if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?",
(kind, folds[0])).fetchone():
raise ValueError(f"there is already a {kind} {name!r}")
place_id = int(conn.execute(
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
(kind, folds[0], name.strip(), parent_id)).lastrowid or 0)
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
[(f, place_id) for f in folds])
gaz = self._gazetteer(conn)
for folded in folds:
gazetteer.relink(conn, gaz, folded)
return place_id
def relink_places(self) -> int:
"""Resolve every role's own places again, after the gazetteer was
seeded anew. Returns how many roles now resolve differently."""
with self.transaction() as conn:
return gazetteer.relink(conn, self._gazetteer(conn))
# ---------------------------------------------------------------- roles
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
@@ -426,19 +573,32 @@ class Store:
alone for good, whatever the plugin's raw guess now says — see
`set_extracted` for the other half of the rule.
`seniority` is the title's level (`seniority.from_title`) until
extract has read the role and extract's from then on, as `function`
is. The classifier is called here rather than in a plugin, so a role
from any source gets it, the way `country` comes from `place` here
too. A level the board states in `raw` is a board field and wins over
both. `place_fold` is written with `place`, under the same keep-filled
rule.
`seniority` is the first filled of three levels. `seniority_model`
is extract's (`set_extracted`) and never written here.
`seniority_stated` is the host's word as `raw["seniority"]` carries
it, already in `SENIORITIES` (anything else is ''), and
`seniority_title` is `seniority.from_title`; both are rewritten every
pull. The classifier is called here rather than in a plugin, so a
role from any source gets it, the way `country` comes from `place`.
The role's places (`role_places`) are the texts of `raw["places"]`,
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
against the gazetteer; `place` is the line shown, the texts joined
when `raw` has none. Like `place`, a pull with no place leaves them
as they are. `place_fold` is written with `place`, under the same
keep-filled rule. `teaser` is kept-filled too.
A role is embedded from its description, else its teaser, else its
title (`app.jobs.embed.document`). When a pull brings a better one of
these than the role had, its vector is dropped so the next `embed`
reads it.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
raw = item.raw or {}
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
place_texts = gazetteer.texts(raw)
place = str(raw.get("place") or "") or "; ".join(place_texts)
stamp = self.now()
published = _iso(item.published_at)
row: dict[str, Any] = {
@@ -463,12 +623,16 @@ class Store:
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row["place"] = place
row["country"] = row["country"] or country_of(place)
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
stated = str(raw.get("seniority") or "")
row["seniority"] = stated or from_title(item.title)
row["seniority_stated"] = stated if stated in SENIORITIES else ""
row["seniority_title"] = from_title(item.title)
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
row["teaser"] = str(raw.get("teaser") or "")
row["place_fold"] = places.fold(row["place"])
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
@@ -476,13 +640,13 @@ class Store:
row["salary_text"] = str(raw.get("salary_text") or "")
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
"feed_text", "original_url", "teaser", *_RAW_TEXT)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text")]
"salary_text", "seniority_stated", "seniority_title")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -493,27 +657,29 @@ class Store:
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
"ELSE roles.function END")
# The title's level follows the title until extract has read the
# role, and is extract's from then on; a board's stated level wins.
updates.append("seniority=excluded.seniority" if stated else
"seniority=CASE WHEN roles.extracted_at IS NULL "
"THEN excluded.seniority ELSE roles.seniority END")
# The model's level, else the host's, else the title's.
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
updates.append("place_fold=CASE WHEN excluded.place != '' "
"THEN excluded.place_fold ELSE roles.place_fold END")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
with self.transaction() as conn:
before = conn.execute("SELECT description FROM roles WHERE uid=?",
before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?",
(row["uid"],)).fetchone()
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
# A role listed without its text is embedded from its title, and
# `embed` encodes a role once. When the text arrives on a later
# pull, as it does for a host that states it on the role's own
# page, the vector goes, so the next `embed` reads the text.
if before is not None and not before["description"] and row["description"]:
if place_texts:
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
# `embed` encodes a role once, from the best text it has. When a
# later pull brings a better one, as a host that states the text
# on the role's own page does, the vector goes, so the next
# `embed` reads it.
if before is not None and _text_rank(row["description"] or before["description"],
row["teaser"] or before["teaser"]) \
> _text_rank(before["description"], before["teaser"]):
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
return before is None
@@ -533,6 +699,7 @@ class Store:
role = ScoredRole.from_row(row)
role.tags = self._tags_of(uid)
role.copies = self._copies_of(uid)
role.places = self.role_places(uid)
return role
def _copies_of(self, uid: str) -> list[RoleCopy]:
@@ -685,7 +852,7 @@ class Store:
`company_keys` keeps those exact companies. `places` keeps a role any
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
`place:malmo`), matched on whole folded words; a token that names
`place:malmo`), over the gazetteer (`_places_sql`); a token that names
nothing is a ValueError. `seniority` keeps the roles at any of those
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
with no level, which is most of them. An empty list is no filter.
@@ -934,8 +1101,8 @@ class Store:
else:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
if places:
# Last, so the cheaper terms have cut the rows it is called on.
clause, values = _places_sql(places)
_known_places(places)
clause, values = _places_sql(places, self.gazetteer())
where.append(clause)
params += values
sql = f"FROM roles {_WITH_VOTES}"
@@ -1087,9 +1254,9 @@ class Store:
non-empty answer overwrites it outright rather than only filling an
empty column — see `upsert_role`'s docstring for the other half of
the rule that then keeps a later pull from putting the guess back.
`seniority` is the other: `upsert_role` fills it from the title, so a
non-empty answer replaces it, unless the board stated a level in
`raw`."""
`seniority` is the other: a non-empty answer is `seniority_model`,
which comes before the stated and the title's level, so it becomes
`seniority` too."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1103,8 +1270,8 @@ class Store:
if function:
sets["function"] = function
level = str(fields.get("seniority") or "")
if level and not json.loads(row["raw"] or "{}").get("seniority"):
sets["seniority"] = level
if level:
sets["seniority_model"] = sets["seniority"] = level
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
@@ -1112,10 +1279,12 @@ class Store:
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets)
filled = len(sets) - ("seniority" in sets)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
gazetteer.write(conn, self._gazetteer(conn), uid,
gazetteer.texts({"place": sets["place"]}))
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
@@ -1375,8 +1544,9 @@ class Store:
as a column so a gated role keeps its row and reason."""
columns: list[str] = []
params: list[Any] = []
gaz = self.gazetteer()
for n, gate in enumerate(gates):
empty, match, values = _gate_sql(gate)
empty, match, values = _gate_sql(gate, gaz)
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values