Merge branch 'wp-e1-places'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:33:07 +02:00
commit 5f0f4ad5b7
10 files changed
+1437 -98

No files matched your search

+6 -2
View File
@@ -2,7 +2,10 @@
The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors`
(see `Store.put_vectors`). What gets encoded is the title and the opening of
the description, which is what says what a role is.
the description, which is what says what a role is; while a role has no
description, the teaser its host's list carried, and while it has neither,
the title alone. `Store.upsert_role` drops the vector when a better text
arrives, so the role is encoded again.
sentence-transformers is optional, as in newsdesk. Without it this task puts
one notice on screen and does nothing; `score` then leaves `fit` out and ranks
@@ -55,7 +58,8 @@ def wants_e5_prefix(model: str) -> bool:
def document(role: Role) -> str:
return f"{role.title}\n{role.description[:BODY_CHARS]}"
"""The title and the description, else the teaser."""
return f"{role.title}\n{(role.description or role.teaser)[:BODY_CHARS]}"
def role_text(model: str, role: Role) -> str:
+73 -16
View File
@@ -12,7 +12,10 @@ from __future__ import annotations
import json
import re
import sys
import unicodedata
from functools import lru_cache
from pathlib import Path
from typing import Annotated, Any, Literal, get_args
from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator
@@ -358,8 +361,9 @@ _REGIONS = ("europe", "emea", "eu", "european union", "eea")
#: state or a Canadian province in the same part of the line beats one of
#: these ("London, Ontario" is Canada); a country name does not lose to it.
_CITIES: dict[str, str] = {
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
**dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck"), "AT"),
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege",
"luik"), "BE"),
**dict.fromkeys(("sofia",), "BG"),
**dict.fromkeys(("zagreb",), "HR"),
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
@@ -369,7 +373,7 @@ _CITIES: dict[str, str] = {
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
**dict.fromkeys(("tallinn", "tartu"), "EE"),
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
**dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"),
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
"lille", "rennes", "amiens"), "FR"),
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
@@ -378,27 +382,30 @@ _CITIES: dict[str, str] = {
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck"), "DE"),
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
**dict.fromkeys(("budapest",), "HU"),
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
"florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples",
"napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"),
**dict.fromkeys(("riga",), "LV"),
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
**dict.fromkeys(("luxembourg", "luxemburg"), "LU"),
**dict.fromkeys(("valletta",), "MT"),
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
"delft"), "NL"),
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
"PL"),
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"),
**dict.fromkeys(("belgrade", "beograd"), "RS"),
**dict.fromkeys(("bratislava",), "SK"),
**dict.fromkeys(("ljubljana",), "SI"),
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
"granada", "santiago de compostela"), "ES"),
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras",
"linkoping", "orebro", "norrkoping", "jonkoping"), "SE"),
**dict.fromkeys(("reykjavik",), "IS"),
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
@@ -408,7 +415,7 @@ _CITIES: dict[str, str] = {
"slough", "southampton", "swindon", "liverpool", "redhill",
"milton keynes"), "GB"),
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
"sankt gallen", "st gallen"), "CH"),
"sankt gallen", "st gallen", "lucerne", "luzern"), "CH"),
**dict.fromkeys(("kyiv", "kiev"), "UA"),
**dict.fromkeys(("istanbul", "ankara"), "TR"),
**dict.fromkeys(("tirana",), "AL"),
@@ -659,9 +666,10 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
# -------------------------------------------------------------- posting dates
#: Hosts whose date is a creation or first-publication timestamp, safe to
#: read as when a role was posted, checked against live boards on
#: 2026-09-29:
#: Hosts whose date is a creation or first-publication timestamp, for a
#: reader that does not declare `READS` yet (see `trusted_posted`); a reader
#: that declares it says so with `fields.posted="trusted"`. Checked against
#: live boards on 2026-09-29:
#:
#: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers
#: `first_published` over `updated_at`. A live board (adyen) showed a job
@@ -708,14 +716,39 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
#: HR Manager) state a date nobody checked, and Pinpoint states none.
#:
#: A host not listed here, or a role whose stored `published_at` fails the
#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the
#: safe default this migration is replacing everywhere else.
TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({
#: sanity check in `posted_at`, keeps `first_seen` as its posted date.
_UNDECLARED_TRUSTED: frozenset[str] = frozenset({
"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
"personio", "teamtailor", "jobindex", "adzuna", "platsbanken",
"workday", "successfactors", "oracle", "phenom", "bamboohr", "jobylon", "rippling",
})
@lru_cache(maxsize=1)
def _declared_posted() -> dict[str, str]:
"""`fields.posted` of every reader that declares `READS`, by name. Read
once: the readers are loaded when `plugins.readers` is imported. `plugins`
is a top-level folder beside `app`, reached as `app/jobs/extract.py`
reaches it."""
root = str(Path(__file__).resolve().parents[1])
if root not in sys.path:
sys.path.insert(0, root)
from plugins.readers import READERS
return {name: module.READS.fields.posted for name, module in READERS.items()
if getattr(module, "READS", None) is not None}
def trusted_posted(host: str) -> bool:
"""Whether `host` states a date that is safe to read as when a role was
posted: its reader declares `fields.posted="trusted"`, or, for a reader
without `READS` (or a search provider not yet a reader), it is in
`_UNDECLARED_TRUSTED`."""
declared = _declared_posted()
if host in declared:
return declared[host] == "trusted"
return host in _UNDECLARED_TRUSTED
#: A `published_at` before this is a parsing slip or a placeholder, not a
#: real posting date: no board here existed before 2015.
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
@@ -732,7 +765,7 @@ def posted_host(raw: dict[str, Any]) -> str:
def posted_at(published_at: str, first_seen: str, host: str) -> str:
"""The date a role counts as posted: the earlier of `first_seen` (when
the app itself first stored it) and `published_at` (what the board
claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and
claims), but only when `trusted_posted(host)` and
`published_at` is neither absurdly old nor later than `first_seen` — a
role cannot be posted after the app already saw it, trusted host or not,
which also catches a `published_at` the board reports in the future
@@ -740,7 +773,7 @@ def posted_at(published_at: str, first_seen: str, host: str) -> str:
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
is exact and needs no parsing. A role with no trusted date, or with one
that fails the check, keeps `first_seen`."""
if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen:
if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen:
return published_at
return first_seen
@@ -775,6 +808,20 @@ class RoleCopy(Model):
posted_at: str = ""
class RolePlace(Model):
"""One place of a role (`role_places`): the host's `text`, and the place
of the gazetteer it resolves to, `place_id` None (and `name` and `kind`
empty) while it resolves to none. `url` is the link of a copy folded into
the role, '' for the role's own places."""
position: int = 0
text: str = ""
place_id: int | None = None
name: str = ""
kind: str = ""
url: str = ""
class Role(Model):
"""A row of `roles`: one posting on one board."""
@@ -797,13 +844,20 @@ class Role(Model):
#: could be normalised into `salary_min`/`salary_max`/`currency`
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
salary_text: str = ""
#: The level shown and filtered on: the first filled of `seniority_model`,
#: `seniority_stated` and `seniority_title`.
seniority: str = ""
seniority_model: str = ""
seniority_stated: str = ""
seniority_title: str = ""
function: str = ""
permit: Permit = ""
#: ISO 639-1 codes the posting says are required, lowercase.
languages: JsonStrings = []
description: str = ""
feed_text: str = ""
#: The short text the host's list carries, until `description` arrives.
teaser: str = ""
original_url: str = ""
published_at: str
#: The date this role counts as posted: see `posted_at`. Used for
@@ -838,6 +892,9 @@ class Role(Model):
#: The open copies of this role, oldest first, when it is the kept one;
#: empty otherwise. Not a column: filled by `Store.role`.
copies: list[RoleCopy] = []
#: Every place of the role, in the host's order. Not a column: filled by
#: `Store.role` from `role_places`.
places: list[RolePlace] = []
@computed_field # type: ignore[prop-decorator]
@property
+313 -5
View File
@@ -19,8 +19,13 @@ several hundred milliseconds a query, as a set intersection on the words of a
line it costs a few. `register` gives a connection that answer as the SQL
function `place_match`, and `match_sql` returns the clause that calls it.
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
chip's text back from the address. This module opens no database; the caller
hands over the places it holds, or its connection.
chip's text back from the address.
The filter and the gates no longer match words: a role's places are resolved
against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
and a token is the places under the place it names. `matches` stays the rule
for a `place:` token the gazetteer does not know. This module opens no
database; the caller hands over the places it holds, or its connection.
"""
from __future__ import annotations
@@ -32,10 +37,12 @@ from collections.abc import Iterable
from dataclasses import dataclass
from functools import lru_cache
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
country_of)
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed",
"Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse",
"placeless", "register", "seed", "suggest", "towns"]
def fold(text: str) -> str:
@@ -538,6 +545,9 @@ _PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
#: Danish boards append "and the possibility of working from home".
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
#: A slash between two places, "Malmö/Lund", "Berlin / Munich".
_SLASH = re.compile(r"\s*/\s*")
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
@@ -665,3 +675,301 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
regions.sort(key=lambda s: (-s.count, s.label))
towns.sort(key=lambda s: (-s.count, s.label))
return (countries + regions + towns)[:limit]
# ------------------------------------------------------------------ gazetteer
#: The kinds of place, widest first: the index is how specific a kind is.
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
#: Entries of `_CITIES` that name a region rather than a town.
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
"bay area"))
#: Words a place text holds when it names how or where-ever a role is worked
#: rather than a place: "Remote", "Multiple Locations", "Home based". A text
#: of these alone is no place, so it is never listed as unresolved.
_PLACELESS = frozenset((
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
def placeless(text: str) -> bool:
"""Whether a place text names no place at all: empty, or only numbers and
words such as `Remote`, `Hybrid`, `2 Locations`."""
return all(word in _PLACELESS or word.isdigit() for word in fold(text).split())
@dataclass(frozen=True)
class Seed:
"""One place as the code knows it: `key` is unique within `kind` (an ISO
code for a country, a region's key, a town's group), `parent` the
`(kind, key)` of the place it lies in, `aliases` the folds it is written
as."""
kind: str
key: str
name: str
parent: tuple[str, str] | None
aliases: tuple[str, ...]
def _subdivisions() -> list[tuple[str, str, str]]:
"""`(country, code, name)` of every US state and Canadian province."""
found = []
for line in _SUBDIVISIONS.split("\n"):
if line:
country, rest = line.split(" ", 1)
for entry in rest.split("|"):
name, code = entry.strip().rsplit(" ", 1)
found.append((country, code, name))
return found
def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
"""The gazetteer the dictionaries in code describe, parents before their
children, and the inclusions beyond the tree (Øresund holds the Capital
Region and Skåne, which lie in two countries).
Europe is a continent over the countries of `_EUROPE`, and is also what
`EU`, `EMEA` and `European Union` name. Every country of `_COUNTRY_NAMES`
by its names; the US states and Canadian provinces as regions; the
regions of `REGIONS`; the towns of the alias groups, of the regions and
of `_CITIES`, one place per group (`Copenhagen`, `København`,
`Kopenhamn` are one), in their region when a region lists them and else
in their country."""
europe = ("continent", "europe")
seeds: dict[tuple[str, str], Seed] = {}
def add(kind: str, key: str, name: str, parent: tuple[str, str] | None,
aliases: Iterable[str]) -> None:
folds = tuple(dict.fromkeys(a for a in aliases if a))
if (kind, key) in seeds:
old = seeds[(kind, key)]
folds = tuple(dict.fromkeys((*old.aliases, *folds)))
seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds)
else:
seeds[(kind, key)] = Seed(kind, key, name, parent, folds)
add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS)))
for code, names in _COUNTRY_LABELS.items():
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
for country, code, name in _subdivisions():
add("region", f"{country}-{code}", name, ("country", country), (fold(name),))
for region in REGIONS.values():
parent = ("country", region.country) if region.country else europe
add("region", region.key, region.label, parent,
(v for name in (region.key, region.label, *region.names)
for v in _variants(fold(name))))
for name in sorted(_CITY_REGIONS):
add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,))
# Towns, one per group, in the region that lists them first.
town_region: dict[str, str] = {}
for region in REGIONS.values():
for name in region.towns:
town_region.setdefault(_group(fold(name)), region.key)
folds: dict[str, dict[str, None]] = {}
shown: dict[str, str] = {}
def add_town(folded: str, name: str) -> None:
key = _group(folded)
folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded)))
shown.setdefault(key, name)
for spellings in _ALIAS_GROUPS:
for spelling in spellings:
add_town(fold(spelling), spellings[0])
for region in REGIONS.values():
for name in region.towns:
add_town(fold(name), name)
for city in _CITIES:
if city not in _CITY_REGIONS and city not in _REGION_BY_NAME:
add_town(city, _TOWN_LABELS.get(city) or city.title())
for key, forms in folds.items():
country = next((_CITIES[f] for f in forms if f in _CITIES), "")
if key in town_region:
parent = ("region", town_region[key])
elif country:
parent = ("country", country)
else:
raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES")
add("city", key, shown[key], parent, forms)
includes = [(("region", region.key), ("region", part))
for region in REGIONS.values() for part in region.includes]
return list(seeds.values()), includes
@dataclass(frozen=True)
class Place:
"""A row of `places`."""
id: int
kind: str
key: str
name: str
parent_id: int | None
class Gazetteer:
"""The places, their aliases and what lies in what, as the database holds
them, read into memory: what resolves a role's place text to places and a
filter token to the places under it. Built by the store from its tables,
and rebuilt when they change."""
def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]],
aliases: Iterable[tuple[str, int]],
includes: Iterable[tuple[int, int]] = ()) -> None:
self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows}
self.by_key: dict[tuple[str, str], int] = {
(p.kind, p.key): p.id for p in self.places.values()}
by_fold: dict[str, list[int]] = {}
for folded, place_id in aliases:
by_fold.setdefault(folded, []).append(place_id)
self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()}
self._longest = max((len(f.split()) for f in self.aliases), default=1)
self._up: dict[int, set[int]] = {}
self._down: dict[int, set[int]] = {}
for p in self.places.values():
if p.parent_id is not None:
self._link(p.parent_id, p.id)
for whole, part in includes:
self._link(whole, part)
self._ancestors: dict[int, frozenset[int]] = {}
self._resolved: dict[str, tuple[int, ...]] = {}
def _link(self, whole: int, part: int) -> None:
self._up.setdefault(part, set()).add(whole)
self._down.setdefault(whole, set()).add(part)
def ancestors(self, place_id: int) -> frozenset[int]:
"""Every place `place_id` lies in, through parents and inclusions."""
if (found := self._ancestors.get(place_id)) is None:
seen: set[int] = set()
todo = list(self._up.get(place_id, ()))
while todo:
up = todo.pop()
if up not in seen:
seen.add(up)
todo += self._up.get(up, ())
found = self._ancestors[place_id] = frozenset(seen)
return found
def under(self, ids: Iterable[int]) -> frozenset[int]:
"""The places and every place that lies in one of them."""
seen: set[int] = set()
todo = list(ids)
while todo:
place_id = todo.pop()
if place_id not in seen:
seen.add(place_id)
todo += self._down.get(place_id, ())
return frozenset(seen)
def country(self, place_id: int) -> str:
"""The ISO code of the country a place is in, '' above or outside one."""
place: Place | None = self.places.get(place_id)
while place is not None and place.kind != "country":
place = self.places.get(place.parent_id) if place.parent_id is not None else None
return place.key if place is not None else ""
def codes(self, ids: Iterable[int]) -> frozenset[str]:
"""The ISO codes of the countries among `ids`, and `EU` with Europe:
what a `roles.country` column holds for a role of one of them."""
found = set()
for place_id in ids:
place = self.places[place_id]
if place.kind == "country":
found.add(place.key)
elif (place.kind, place.key) == ("continent", "europe"):
found.add("EU")
return frozenset(found)
def _specific(self, place_id: int) -> int:
return KINDS.index(self.places[place_id].kind)
def _hits(self, folded: str) -> list[tuple[int, ...]]:
words = folded.split()
hits = []
i = 0
while i < len(words):
for n in range(min(self._longest, len(words) - i), 0, -1):
if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None:
hits.append(ids)
i += n
break
else:
i += 1
return hits
def resolve(self, text: str) -> tuple[int, ...]:
"""The places one place text names, most specific first from the left:
`Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund`
is Malmö and Lund, `Paderborn, Germany` is Germany, `Remote` is none.
Every alias the text holds as whole words is a candidate, longest
first. A fold that names several places is the one in the country
`country_of` reads off the text, if one is, and else the most specific
(`Luxembourg` the town before the country). A candidate that another
one lies in is dropped, and so is one outside the one country the text
names, or, naming none, outside the regions it names in one country:
`London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the
Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several
countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country
`country_of` reads (from a state code, `Paris, TX`, or a code in
parentheses, `Remote (DE)`), the text is that country."""
folded = fold(text)
if (found := self._resolved.get(folded)) is None:
parts = _SLASH.split(_AND_HOME.sub("", text or ""))
found = tuple(dict.fromkeys(
p for part in parts for sub in _PART.split(part) for p in self._resolve(sub)))
self._resolved[folded] = found
return found
def _resolve(self, text: str) -> list[int]:
"""`resolve` for one part of a text, cut where `or`, `/` or `;` end one
place."""
code = country_of(text)
country = self.by_key.get(("country", code)) if code and code != "EU" else None
within = self.under((country,)) if country is not None else frozenset()
picked: list[int] = []
for hit in self._hits(fold(text)):
choice = [p for p in hit if p in within] or list(hit)
best = max(choice, key=self._specific)
if best not in picked:
picked.append(best)
# The widest units named hold the rest: one country, else the regions
# of one country. Several countries are a list, not a path.
for kind in ("country", "region"):
named = [p for p in picked if self.places[p].kind == kind]
if len({self.country(p) for p in named}) == 1:
holds = self.under(named)
picked = [p for p in picked if p in holds or self.places[p].kind == "continent"
or any(p in self.ancestors(q) for q in named)]
if named:
break
wider = {a for p in picked for a in self.ancestors(p)}
picked = [p for p in picked if p not in wider]
if country is not None and not any(p in within or p in self.ancestors(country)
for p in picked):
picked = [country]
return picked
def lookup(self, token: str) -> frozenset[int] | None:
"""The places a filter token names, before what lies in them: `country:DK`
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
every place written so (a town, or a region or a country by name).
None when the gazetteer knows no such place."""
tok = parse(token)
if tok.kind == "country":
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
found = {self.by_key[key]} if key in self.by_key else set()
elif tok.kind == "region":
found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \
else set(self.aliases.get(tok.value, ()))
else:
found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())}
return frozenset(found) or None
+2
View File
@@ -3,6 +3,8 @@
schema.py the tables, and the numbered migrations that bring an older
file to the current shape
store.py `Store`, every SQL statement in the application
gazetteer.py the places tables: seeded from the code, read back, and a
role's places written and relinked
Written for roledesk rather than imported from newsdesk: `desk` has no storage
package on purpose, and the connection plumbing here is a deliberate copy.
+121
View File
@@ -0,0 +1,121 @@
"""The gazetteer's tables: seeding them from the code, reading them back, and
writing a role's places.
`places`, `place_aliases` and `place_includes` hold the places
(`app.places.seed` on a new or migrated file, and what a person adds later).
`role_places` holds one row per place of a role: the host's own text, its
fold, and the place it resolves to, NULL for a text the gazetteer does not
know. A text that names several places ("Malmö/Lund") is one row per place,
each with the same `position` and `text`. A row with a `url` is the place of
a copy folded into the role (`app/dupes.py`) and is not the role's own: only
own rows (`url = ''`) are rewritten here.
"""
from __future__ import annotations
import sqlite3
from collections.abc import Iterable, Mapping, Sequence
from typing import Any
from app import places
#: What changes when the gazetteer does: rows are only ever added.
VERSION = ("SELECT (SELECT MAX(rowid) FROM places), (SELECT MAX(rowid) FROM place_aliases), "
"(SELECT COUNT(*) FROM place_includes)")
def seed(conn: sqlite3.Connection) -> int:
"""Insert what `places.seed` describes, leaving rows already there alone.
Returns how many places were added."""
seeds, includes = places.seed()
ids = {(kind, key): place_id for place_id, kind, key in
conn.execute("SELECT id, kind, key FROM places")}
added = 0
for s in seeds:
if (s.kind, s.key) not in ids:
parent = ids[s.parent] if s.parent else None
ids[(s.kind, s.key)] = conn.execute(
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
(s.kind, s.key, s.name, parent)).lastrowid
added += 1
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
[(a, ids[(s.kind, s.key)]) for a in s.aliases])
conn.executemany("INSERT OR IGNORE INTO place_includes (place_id, part_id) VALUES (?, ?)",
[(ids[whole], ids[part]) for whole, part in includes])
return added
def load(conn: sqlite3.Connection) -> places.Gazetteer:
return places.Gazetteer(
[tuple(r) for r in conn.execute("SELECT id, kind, key, name, parent_id FROM places")],
[tuple(r) for r in conn.execute("SELECT fold, place_id FROM place_aliases")],
[tuple(r) for r in conn.execute("SELECT place_id, part_id FROM place_includes")])
def texts(raw: Mapping[str, Any]) -> list[str]:
"""A role's place texts from `Item.raw`: the `text` of each of `places`
(`[{"text": ..., "remote": ...}]`, the reader contract), else `place` split
at `;`, which is how the older readers join several places into one line.
Blank and repeated texts are dropped."""
given = raw.get("places")
if isinstance(given, list) and given:
found = [str((p.get("text") if isinstance(p, Mapping) else p) or "") for p in given]
else:
found = str(raw.get("place") or "").split(";")
return list(dict.fromkeys(t.strip() for t in found if t and t.strip()))
def rows(gaz: places.Gazetteer, uid: str, place_texts: Sequence[str]
) -> list[tuple[str, int, str, str, int | None]]:
"""`(uid, position, text, fold, place_id)` for each place of the texts."""
found: list[tuple[str, int, str, str, int | None]] = []
for position, text in enumerate(place_texts):
folded = places.fold(text)
found += [(uid, position, text, folded, place_id)
for place_id in (gaz.resolve(text) or (None,))]
return found
_INSERT = "INSERT INTO role_places (uid, position, text, fold, place_id) VALUES (?, ?, ?, ?, ?)"
def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
place_texts: Sequence[str]) -> None:
"""Replace one role's own places with these texts, resolved."""
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
"""`write` for many roles in one statement each way. Returns rows written."""
batch = list(by_uid)
conn.executemany("DELETE FROM role_places WHERE uid = ? AND url = ''",
[(uid,) for uid, _ in batch])
new = [row for uid, found in batch for row in rows(gaz, uid, found)]
conn.executemany(_INSERT, new)
return len(new)
def relink(conn: sqlite3.Connection, gaz: places.Gazetteer, words: str | None = None) -> int:
"""Resolve the own places of every role again, or only of the roles with a
place text that holds `words` (a fold) as whole words, after an alias or a
place was added. Returns how many roles now resolve differently."""
sql = "SELECT uid, position, text, place_id FROM role_places WHERE url = ''"
params: list[Any] = []
if words:
sql += " AND ' ' || fold || ' ' LIKE ? ESCAPE '\\'"
escaped = words.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
params.append(f"% {escaped} %")
sql = ("SELECT uid, position, text, place_id FROM role_places WHERE url = '' AND uid IN "
f"(SELECT uid FROM ({sql}))")
before: dict[str, dict[tuple[int, str], set[int | None]]] = {}
for uid, position, text, place_id in conn.execute(sql + " ORDER BY uid, position", params):
before.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
changed = []
for uid, held in before.items():
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in held}
if now != held:
changed.append((uid, [text for _, text in sorted(held)]))
write_many(conn, gaz, changed)
return len(changed)
+156 -7
View File
@@ -18,8 +18,9 @@ import sqlite3
from collections.abc import Callable
from app import places
from app.models import country_of, posted_at, posted_host
from app.models import SENIORITIES, country_of, posted_at, posted_host
from app.seniority import from_title
from app.storage import gazetteer
log = logging.getLogger(__name__)
@@ -44,8 +45,9 @@ CREATE TABLE IF NOT EXISTS roles (
-- require the country: "UK | Remote" is not remote from Berlin.
place TEXT NOT NULL DEFAULT '',
place_key TEXT NOT NULL DEFAULT '',
-- places.fold(place): what the Place filter and the place gate match
-- whole words against, through the place_match function (app/places.py).
-- places.fold(place): what a place: token the gazetteer does not know is
-- matched against as whole words, through place_match (app/places.py).
-- The filter and the gates otherwise read role_places.
place_fold TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
@@ -53,7 +55,8 @@ CREATE TABLE IF NOT EXISTS roles (
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
-- seniority.from_title(title) at ingest, until extract reads the posting.
-- The level shown and filtered on: the first filled of seniority_model,
-- seniority_stated and seniority_title (the last columns), written with them.
seniority TEXT NOT NULL DEFAULT '',
function TEXT NOT NULL DEFAULT '',
-- Whether the posting says it sponsors a work permit, and the languages
@@ -69,7 +72,7 @@ CREATE TABLE IF NOT EXISTS roles (
-- stored the role; posted_at (app.models.posted_at) is the earlier of
-- first_seen and published_at, trusted only for a host known to report a
-- creation date rather than one that moves on an edit
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
-- (app.models.trusted_posted) — recency and the feed's newest order
-- read posted_at, first_seen stays "when the app found it". closed_at is
-- set when a board fetched in full no longer lists the role, and cleared
-- if it returns.
@@ -100,7 +103,62 @@ CREATE TABLE IF NOT EXISTS roles (
-- precedes, which the migration tests do.
dup_of TEXT,
created_at TEXT NOT NULL,
salary_text TEXT NOT NULL DEFAULT ''
salary_text TEXT NOT NULL DEFAULT '',
teaser TEXT NOT NULL DEFAULT '',
seniority_model TEXT NOT NULL DEFAULT '',
seniority_stated TEXT NOT NULL DEFAULT '',
seniority_title TEXT NOT NULL DEFAULT ''
);
-- roles.teaser is the short text a list carries, kept until the full text
-- arrives and embedded while there is none; a later empty teaser does not
-- wipe it. seniority_model, seniority_stated and seniority_title are the
-- three levels behind roles.seniority: the model's (extract), the host's own
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
-- among the last columns: SQLite cannot drop a column a comment precedes,
-- which the migration tests do.
-- The gazetteer: a place is a continent, a country (key the ISO code), a
-- region or a city, in the place parent_id names. key is unique per kind:
-- a region's key (app.places.REGIONS), a town's group. Seeded from the
-- dictionaries in app/models.py and app/places.py (app.places.seed).
CREATE TABLE IF NOT EXISTS places (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
key TEXT NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER REFERENCES places(id),
UNIQUE (kind, key)
);
-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn
-- and copenhagen are Copenhagen. One fold may name several places
-- (Luxembourg the town and the country).
CREATE TABLE IF NOT EXISTS place_aliases (
fold TEXT NOT NULL,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
UNIQUE (fold, place_id)
);
-- What a place holds beyond its children: Øresund holds the Capital Region
-- of Denmark and Skåne, which lie in two countries.
CREATE TABLE IF NOT EXISTS place_includes (
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (place_id, part_id)
);
-- One row per place of a role (app/storage/gazetteer.py): the host's text in
-- the host's order (position), its fold, and the place it resolves to, NULL
-- while the gazetteer does not know it. A text naming several places is a
-- row for each. url is the link of a copy folded into this role, '' for
-- the role's own places.
CREATE TABLE IF NOT EXISTS role_places (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
position INTEGER NOT NULL,
text TEXT NOT NULL,
fold TEXT NOT NULL,
place_id INTEGER REFERENCES places(id),
url TEXT NOT NULL DEFAULT ''
);
-- The open vocabulary summarise reads off a posting: the role family, the
@@ -303,6 +361,11 @@ CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
-- Partial: all but about 3% of roles are no copy.
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
ON roles(dup_of) WHERE dup_of IS NOT NULL;
-- A role's places, and the roles in a set of places (the Place filter and
-- the country and place gates, `uid IN (... WHERE place_id IN ...)`).
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -469,6 +532,10 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
# A Python step: the first rebuild runs the rule in app/dupes.py.
(18, "roles.dup_of: one posting listed several times is one role, the others "
"copies of it", lambda conn: _add_dup_of(conn)),
# A Python step: the seed and the resolver are Python.
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
"places, resolved against a gazetteer; roles.teaser; the three seniority "
"sources", lambda conn: _add_places(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -592,6 +659,81 @@ def _add_dup_of(conn: sqlite3.Connection) -> None:
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
_V19 = """
ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT '';
ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT '';
CREATE TABLE places (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
key TEXT NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER REFERENCES places(id),
UNIQUE (kind, key)
);
CREATE TABLE place_aliases (
fold TEXT NOT NULL,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
UNIQUE (fold, place_id)
);
CREATE TABLE place_includes (
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (place_id, part_id)
);
CREATE TABLE role_places (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
position INTEGER NOT NULL,
text TEXT NOT NULL,
fold TEXT NOT NULL,
place_id INTEGER REFERENCES places(id),
url TEXT NOT NULL DEFAULT ''
);
"""
def _add_places(conn: sqlite3.Connection) -> None:
"""Migration 19's step. The tables and columns, the gazetteer seeded, and
every role's places: the `places` of its `raw` where a reader stated
them, else its `place` column cut at `;` (not `raw`'s, which a pull that
read the role's own page sends empty), each resolved once per distinct
text.
The level splits into its sources without changing: on a role extract
has read, `seniority` is the model's; on any other, it is the stated
level when `raw` states one in the vocabulary, else the title's."""
for statement in _statements(_V19):
conn.execute(statement)
gazetteer.seed(conn)
gaz = gazetteer.load(conn)
rows = conn.execute(
"SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall()
by_uid: list[tuple[str, list[str]]] = []
levels = []
for uid, raw, place, title, level, extracted_at in rows:
try:
given = json.loads(raw or "{}")
except ValueError:
given = {}
given = given if isinstance(given, dict) else {}
if found := gazetteer.texts({"places": given.get("places"), "place": place}):
by_uid.append((uid, found))
stated = str(given.get("seniority") or "")
stated = stated if stated in SENIORITIES else ""
if extracted_at is not None:
levels.append((level, stated, from_title(title) if level else "", uid))
elif stated:
levels.append(("", stated, from_title(title), uid))
elif level:
levels.append(("", "", level, uid))
written = gazetteer.write_many(conn, gaz, by_uid)
conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, "
"seniority_title = ? WHERE uid = ?", levels)
log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d",
len(gaz.places), written, len(by_uid), len(levels))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
@@ -659,6 +801,13 @@ def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
conn.execute("COMMIT")
def _create(conn: sqlite3.Connection) -> None:
"""A new file: `SCHEMA`, and the gazetteer seeded."""
for statement in _statements(SCHEMA):
conn.execute(statement)
gazetteer.seed(conn)
def initialise(conn: sqlite3.Connection) -> list[int]:
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
fresh = conn.execute(
@@ -668,7 +817,7 @@ def initialise(conn: sqlite3.Connection) -> list[int]:
if fresh:
# Atomic like a migration: a file half created would otherwise carry
# the marker table and be taken for one to migrate.
_apply(conn, LATEST, SCHEMA)
_apply(conn, LATEST, _create)
else:
applied = migrate(conn)
for statement in _statements(INDEXES):
+225 -55
View File
@@ -22,14 +22,14 @@ import numpy as np
from desk.models import Item
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
ScoredRole,
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
)
from app.seniority import from_title
from app.storage import schema
from app.storage import gazetteer, schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
@@ -101,7 +101,8 @@ _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
#: one, else the description's opening. A `SUBSTR` of the source column, not
#: the column itself, so a big `description` never leaves SQLite whole for a
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
_EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), "
"roles.teaser)")
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
@@ -177,23 +178,34 @@ def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
#: A role with no place the gazetteer resolved, over `roles r`.
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
"""One gate as `(empty, match, params)`: SQL that is true when the role's
column is empty, and SQL that is true when it matches, over `roles r`.
`params` belong to `match`; `empty` takes none."""
`params` belong to `match`; `empty` takes none.
`country` and `place` match over the gazetteer, as the feed's Place
filter does (`_places_sql`): a country gate is met by a role whose
`country` is one of its codes or that has a place in one of them, so
`DK` is met by `Copenhagen; London`. Its column is empty only when the
role has neither. A place gate's value is a place token, bare text a
`place:`."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "country":
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
codes=[v.upper() for v in values])
return f"(r.country = '' AND {_UNPLACED})", match, params
if gate.field == "place":
# Whole folded words, as the feed's Place filter: `lund` is not in
# `Billund`. A value is a place token, bare text a `place:`.
clauses = [places.match_sql(v, place="r.place_fold", country="r.country")
for v in values]
match = " OR ".join(sql for sql, _ in clauses) or "0"
return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps]
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
@@ -205,18 +217,54 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]:
"""The Place filter over `roles`: any of the tokens, each a `place_match`
call (`places.match_sql`). ValueError for a token that names nothing,
which would otherwise match no role without saying why."""
clauses, params = [], []
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
"""Any of the place tokens over `table`, as a join over the gazetteer: a
role with a place under one of the places the tokens name (`country:DK`
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
Malmö by any spelling), or whose `country` is one of their countries (a
role whose board gave a country and no place). `codes` are more values of
`country` to accept. A `place:` token the gazetteer does not know is
matched on whole words of the role's folded line (`places.match_sql`),
so a town nobody has entered yet is still found."""
ids: set[int] = set()
found = {c for c in codes if c}
clauses: list[str] = []
params: list[Any] = []
for token in tokens:
named = gaz.lookup(token)
if named is None:
sql, values = places.match_sql(token, place=f"{table}.place_fold",
country=f"{table}.country")
clauses.append(sql)
params += values
continue
under = gaz.under(named)
ids |= under
found |= gaz.codes(under)
if ids:
clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places "
"WHERE place_id IN (SELECT value FROM json_each(?)))")
params.insert(0, json.dumps(sorted(ids)))
if found:
clauses.insert(1 if ids else 0,
f"{table}.country IN (SELECT value FROM json_each(?))")
params.insert(1 if ids else 0, json.dumps(sorted(found)))
return f"({' OR '.join(clauses) or '0'})", params
def _known_places(tokens: Sequence[str]) -> None:
"""ValueError for a token that names nothing, which would otherwise match
no role without saying why."""
for token in tokens:
if not places.known(token):
raise ValueError(f"{token!r} is not a place this app knows")
sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country")
clauses.append(sql)
params += values
return f"({' OR '.join(clauses)})", params
def _text_rank(description: str, teaser: str) -> int:
"""What a role is embedded from, as a rank: 2 its description, 1 its
teaser, 0 its title alone."""
return 2 if description else 1 if teaser else 0
def _int_or_none(value: Any) -> int | None:
@@ -351,6 +399,8 @@ class Store:
self.path.parent.mkdir(parents=True, exist_ok=True)
#: What "now" is. Replaced by tests that need two writes a second apart.
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
#: The gazetteer as last read, with the `gazetteer.VERSION` it was read at.
self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
@@ -364,7 +414,7 @@ class Store:
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
registered on every one, since the place filter and the place gate
call it (`places.match_sql`)."""
call it for a town the gazetteer does not know (`_places_sql`)."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
places.register(conn)
@@ -400,6 +450,103 @@ class Store:
def now(self) -> str:
return _iso(self.clock())
# ------------------------------------------------------------- gazetteer
def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer:
"""The gazetteer, read again only when its tables have grown, which
another process may have done: one cheap query a call."""
version = tuple(conn.execute(gazetteer.VERSION).fetchone())
if self._gaz is None or self._gaz[0] != version:
self._gaz = (version, gazetteer.load(conn))
return self._gaz[1]
def gazetteer(self) -> places.Gazetteer:
with closing(self._connect()) as conn:
return self._gazetteer(conn)
def role_places(self, uid: str) -> list[RolePlace]:
"""One role's places in the host's order, a copy's after the role's own."""
return [RolePlace.from_row(r) for r in self._rows(
"""SELECT rp.position, rp.text, rp.place_id, rp.url,
COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind
FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id
WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""",
(uid,))]
def unresolved_places(self, open_only: bool = True,
limit: int | None = None) -> list[tuple[str, str, int]]:
"""`(fold, text, roles)` for every place text no place of the
gazetteer resolves, most roles first: the list a person works through
with `add_place_alias` and `add_place`. `text` is one spelling of the
fold. A text that names no place ("Remote", `places.placeless`) is
left out; with `open_only`, so are closed roles and copies."""
where = "rp.place_id IS NULL"
if open_only:
where += " AND r.closed_at IS NULL AND r.dup_of IS NULL"
rows = self._rows(
f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n
FROM role_places rp JOIN roles r ON r.uid = rp.uid
WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""")
found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows
if not places.placeless(r["fold"])]
return found[:limit] if limit is not None else found
def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int:
"""A place by id, or by a token that names exactly one place."""
if isinstance(place, int):
if place not in gaz.places:
raise ValueError(f"there is no place {place}")
return place
named = gaz.lookup(place) or frozenset()
if len(named) != 1:
raise ValueError(f"{place!r} names {len(named)} places, not one")
return next(iter(named))
def add_place_alias(self, text: str, place: int | str) -> int:
"""Write `text` as a spelling of `place` (an id, or a token such as
`place:copenhagen` or `country:DK` naming one place), and resolve again
every role with a place text that holds it. Returns how many roles now
resolve differently."""
folded = places.fold(text)
if not folded:
raise ValueError("an alias needs a letter or a digit")
with self.transaction() as conn:
place_id = self._place_id(self._gazetteer(conn), place)
conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
(folded, place_id))
return gazetteer.relink(conn, self._gazetteer(conn), folded)
def add_place(self, name: str, kind: str, parent: int | str,
aliases: Sequence[str] = ()) -> int:
"""Add a city or a region the gazetteer lacks, in `parent` (an id or a
token), written as `name` and each of `aliases`, and resolve again the
roles with a place text that holds one of them. Returns its id."""
if kind not in ("city", "region"):
raise ValueError(f"a place added by hand is a city or a region, not {kind!r}")
folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f))
if not folds:
raise ValueError("a place needs a name")
with self.transaction() as conn:
parent_id = self._place_id(self._gazetteer(conn), parent)
if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?",
(kind, folds[0])).fetchone():
raise ValueError(f"there is already a {kind} {name!r}")
place_id = int(conn.execute(
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
(kind, folds[0], name.strip(), parent_id)).lastrowid or 0)
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
[(f, place_id) for f in folds])
gaz = self._gazetteer(conn)
for folded in folds:
gazetteer.relink(conn, gaz, folded)
return place_id
def relink_places(self) -> int:
"""Resolve every role's own places again, after the gazetteer was
seeded anew. Returns how many roles now resolve differently."""
with self.transaction() as conn:
return gazetteer.relink(conn, self._gazetteer(conn))
# ---------------------------------------------------------------- roles
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
@@ -426,19 +573,32 @@ class Store:
alone for good, whatever the plugin's raw guess now says — see
`set_extracted` for the other half of the rule.
`seniority` is the title's level (`seniority.from_title`) until
extract has read the role and extract's from then on, as `function`
is. The classifier is called here rather than in a plugin, so a role
from any source gets it, the way `country` comes from `place` here
too. A level the board states in `raw` is a board field and wins over
both. `place_fold` is written with `place`, under the same keep-filled
rule.
`seniority` is the first filled of three levels. `seniority_model`
is extract's (`set_extracted`) and never written here.
`seniority_stated` is the host's word as `raw["seniority"]` carries
it, already in `SENIORITIES` (anything else is ''), and
`seniority_title` is `seniority.from_title`; both are rewritten every
pull. The classifier is called here rather than in a plugin, so a
role from any source gets it, the way `country` comes from `place`.
The role's places (`role_places`) are the texts of `raw["places"]`,
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
against the gazetteer; `place` is the line shown, the texts joined
when `raw` has none. Like `place`, a pull with no place leaves them
as they are. `place_fold` is written with `place`, under the same
keep-filled rule. `teaser` is kept-filled too.
A role is embedded from its description, else its teaser, else its
title (`app.jobs.embed.document`). When a pull brings a better one of
these than the role had, its vector is dropped so the next `embed`
reads it.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
raw = item.raw or {}
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
place_texts = gazetteer.texts(raw)
place = str(raw.get("place") or "") or "; ".join(place_texts)
stamp = self.now()
published = _iso(item.published_at)
row: dict[str, Any] = {
@@ -463,12 +623,16 @@ class Store:
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row["place"] = place
row["country"] = row["country"] or country_of(place)
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
stated = str(raw.get("seniority") or "")
row["seniority"] = stated or from_title(item.title)
row["seniority_stated"] = stated if stated in SENIORITIES else ""
row["seniority_title"] = from_title(item.title)
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
row["teaser"] = str(raw.get("teaser") or "")
row["place_fold"] = places.fold(row["place"])
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
@@ -476,13 +640,13 @@ class Store:
row["salary_text"] = str(raw.get("salary_text") or "")
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
"feed_text", "original_url", "teaser", *_RAW_TEXT)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text")]
"salary_text", "seniority_stated", "seniority_title")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -493,27 +657,29 @@ class Store:
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
"ELSE roles.function END")
# The title's level follows the title until extract has read the
# role, and is extract's from then on; a board's stated level wins.
updates.append("seniority=excluded.seniority" if stated else
"seniority=CASE WHEN roles.extracted_at IS NULL "
"THEN excluded.seniority ELSE roles.seniority END")
# The model's level, else the host's, else the title's.
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
updates.append("place_fold=CASE WHEN excluded.place != '' "
"THEN excluded.place_fold ELSE roles.place_fold END")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
with self.transaction() as conn:
before = conn.execute("SELECT description FROM roles WHERE uid=?",
before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?",
(row["uid"],)).fetchone()
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
# A role listed without its text is embedded from its title, and
# `embed` encodes a role once. When the text arrives on a later
# pull, as it does for a host that states it on the role's own
# page, the vector goes, so the next `embed` reads the text.
if before is not None and not before["description"] and row["description"]:
if place_texts:
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
# `embed` encodes a role once, from the best text it has. When a
# later pull brings a better one, as a host that states the text
# on the role's own page does, the vector goes, so the next
# `embed` reads it.
if before is not None and _text_rank(row["description"] or before["description"],
row["teaser"] or before["teaser"]) \
> _text_rank(before["description"], before["teaser"]):
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
return before is None
@@ -533,6 +699,7 @@ class Store:
role = ScoredRole.from_row(row)
role.tags = self._tags_of(uid)
role.copies = self._copies_of(uid)
role.places = self.role_places(uid)
return role
def _copies_of(self, uid: str) -> list[RoleCopy]:
@@ -685,7 +852,7 @@ class Store:
`company_keys` keeps those exact companies. `places` keeps a role any
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
`place:malmo`), matched on whole folded words; a token that names
`place:malmo`), over the gazetteer (`_places_sql`); a token that names
nothing is a ValueError. `seniority` keeps the roles at any of those
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
with no level, which is most of them. An empty list is no filter.
@@ -934,8 +1101,8 @@ class Store:
else:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
if places:
# Last, so the cheaper terms have cut the rows it is called on.
clause, values = _places_sql(places)
_known_places(places)
clause, values = _places_sql(places, self.gazetteer())
where.append(clause)
params += values
sql = f"FROM roles {_WITH_VOTES}"
@@ -1087,9 +1254,9 @@ class Store:
non-empty answer overwrites it outright rather than only filling an
empty column — see `upsert_role`'s docstring for the other half of
the rule that then keeps a later pull from putting the guess back.
`seniority` is the other: `upsert_role` fills it from the title, so a
non-empty answer replaces it, unless the board stated a level in
`raw`."""
`seniority` is the other: a non-empty answer is `seniority_model`,
which comes before the stated and the title's level, so it becomes
`seniority` too."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1103,8 +1270,8 @@ class Store:
if function:
sets["function"] = function
level = str(fields.get("seniority") or "")
if level and not json.loads(row["raw"] or "{}").get("seniority"):
sets["seniority"] = level
if level:
sets["seniority_model"] = sets["seniority"] = level
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
@@ -1112,10 +1279,12 @@ class Store:
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets)
filled = len(sets) - ("seniority" in sets)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
gazetteer.write(conn, self._gazetteer(conn), uid,
gazetteer.texts({"place": sets["place"]}))
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
@@ -1375,8 +1544,9 @@ class Store:
as a column so a gated role keeps its row and reason."""
columns: list[str] = []
params: list[Any] = []
gaz = self.gazetteer()
for n, gate in enumerate(gates):
empty, match, values = _gate_sql(gate)
empty, match, values = _gate_sql(gate, gaz)
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values
+5 -1
View File
@@ -164,13 +164,17 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
store.upsert_role(item)
store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))])
with sqlite3.connect(path) as conn:
for table in ("role_places", "place_includes", "place_aliases", "places"):
conn.execute(f"DROP TABLE {table}")
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
conn.execute("PRAGMA user_version = 17")
migrated = Store(path)
assert migrated.migrations_applied == [18]
assert migrated.migrations_applied == [18, 19]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+397
View File
@@ -0,0 +1,397 @@
"""The gazetteer: its seed, the resolver, and a role's places in the store.
The store tests go through `Store`, as the feed and the gates do. The
real-line test compares the join over the gazetteer with the whole-word rule
it replaced (`places.matches`) on the live lines `test_places` holds: the
join must find every role the word rule found, but for the lines listed where
the word rule was wrong.
"""
from __future__ import annotations
import sqlite3
from collections import Counter
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
import pytest
from desk.models import Item
from app import places
from app.models import Gate, _CITIES, _COUNTRY_NAMES
from app.places import Gazetteer, fold, seed
from app.storage import Store
from test_places import LINES, ORACLE_TOKENS
def gazetteer() -> Gazetteer:
seeds, includes = seed()
ids: dict[tuple[str, str], int] = {}
rows, aliases = [], []
for n, s in enumerate(seeds, 1):
ids[(s.kind, s.key)] = n
rows.append((n, s.kind, s.key, s.name, ids[s.parent] if s.parent else None))
aliases += [(a, n) for a in s.aliases]
return Gazetteer(rows, aliases, [(ids[a], ids[b]) for a, b in includes])
GAZ = gazetteer()
def names(text: str) -> list[str]:
return [GAZ.places[p].name for p in GAZ.resolve(text)]
# ---------------------------------------------------------------------- seed
def test_the_seed_holds_the_dictionaries_in_code() -> None:
seeds, includes = seed()
kinds = Counter(s.kind for s in seeds)
assert kinds["continent"] == 1
assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line)
keys = {(s.kind, s.key) for s in seeds}
assert {("region", key) for key in places.REGIONS} <= keys
assert {("region", "oresund"), ("region", "skane")} in [set(i) for i in includes]
folds = {a for s in seeds for a in s.aliases}
assert set(_CITIES) - {"region hovedstaden"} <= folds
assert {"kobenhavn", "kopenhamn", "copenhagen", "skane", "emea", "eu"} <= folds
def test_parents_come_before_their_children() -> None:
seen: set[tuple[str, str]] = set()
for s in seed()[0]:
assert s.parent is None or s.parent in seen, s
seen.add((s.kind, s.key))
def test_copenhagen_is_in_the_capital_region_in_denmark_in_europe() -> None:
copenhagen = GAZ.by_key[("city", "copenhagen")]
above = {GAZ.places[p].key for p in GAZ.ancestors(copenhagen)}
assert above == {"hovedstaden", "oresund", "DK", "europe"}
assert GAZ.country(copenhagen) == "DK"
assert GAZ.by_key[("country", "US")] not in GAZ.under({GAZ.by_key[("continent", "europe")]})
# ------------------------------------------------------------------ resolving
@pytest.mark.parametrize("text, expected", [
("Aarhus N, Region Central Jutland, Denmark", ["Aarhus"]),
("København K", ["Copenhagen"]),
("Copenhagen Office", ["Copenhagen"]),
("Malmö/Lund", ["Malmö", "Lund"]),
("Aalborg, Copenhagen, Kolding", ["Aalborg", "Copenhagen", "Kolding"]),
("Paderborn, Germany", ["Germany"]),
("Skåne län, Sweden", ["Skåne"]),
("London, Ontario", ["Ontario"]),
("Bergen op Zoom, Netherlands", ["Netherlands"]),
("Mexicali, BAJA CALIFORNIA, Mexico", ["Mexico"]),
("Paris, TX", ["United States"]),
("Remote (DE)", ["Germany"]),
("London OR Paris OR Germany", ["London", "Paris", "Germany"]),
("Allerød or Stockholm (SE)", ["Allerød", "Stockholm"]),
("Finland, København, Litauen, Polen, Sverige", ["Finland", "Copenhagen", "Sweden"]),
("Remote, EU or US", ["Europe", "United States"]),
("Home based - EMEA", ["Europe"]),
("Luxembourg", ["Luxembourg"]),
("Nordhavn og mulighed for hjemmearbejde", ["Nordhavn"]),
("Remote", []),
("Oberkochen", []),
("", []),
])
def test_a_text_resolves_to_the_places_it_names(text: str, expected: list[str]) -> None:
assert names(text) == expected
def test_luxembourg_the_town_is_in_luxembourg_the_country() -> None:
town = GAZ.by_key[("city", "luxembourg")]
assert GAZ.resolve("Luxembourg") == (town,)
assert GAZ.country(town) == "LU"
@pytest.mark.parametrize("text", ["Remote", "Hybrid", "Multiple Locations", "2 Locations",
"Remote job", "", " "])
def test_a_text_of_how_a_role_is_worked_is_no_place(text: str) -> None:
assert places.placeless(text)
@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Jena"])
def test_a_text_with_a_name_is_a_place(text: str) -> None:
assert not places.placeless(text)
# -------------------------------------------------------------------- tokens
def test_a_token_names_places_before_what_lies_in_them() -> None:
assert GAZ.lookup("country:DK") == {GAZ.by_key[("country", "DK")]}
assert GAZ.lookup("country:EU") == {GAZ.by_key[("continent", "europe")]}
assert GAZ.lookup("region:Skåne") == {GAZ.by_key[("region", "skane")]}
assert GAZ.lookup("place:kobenhavn") == {GAZ.by_key[("city", "copenhagen")]}
assert GAZ.lookup("place:denmark") == {GAZ.by_key[("country", "DK")]}
assert GAZ.lookup("place:oberkochen") is None
assert GAZ.lookup("country:ZZ") is None
def test_europe_is_every_european_country_and_eu() -> None:
under = GAZ.under(GAZ.lookup("country:EU") or ())
codes = GAZ.codes(under)
assert {"DK", "SE", "DE", "GB", "CH", "EU"} <= codes
assert not {"US", "CN"} & codes
def test_oresund_holds_both_capital_regions() -> None:
under = GAZ.under(GAZ.lookup("region:oresund") or ())
assert {GAZ.by_key[("city", "malmo")], GAZ.by_key[("city", "copenhagen")]} <= under
assert GAZ.by_key[("city", "aarhus")] not in under
# --------------------------------------------------------------------- store
@pytest.fixture
def store(tmp_path: Path) -> Iterator[Store]:
yield Store(tmp_path / "roledesk.db")
def posting(source_id: str, *, title: str = "Backend Engineer", body: str = "",
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw=dict(raw))
item.source = "ats"
return item
def texts(store: Store, uid: str) -> list[tuple[int, str, str]]:
return [(p.position, p.text, p.name) for p in store.role_places(uid)]
def test_a_fresh_file_is_seeded(store: Store) -> None:
gaz = store.gazetteer()
assert len(gaz.places) == len(GAZ.places)
assert ("city", "copenhagen") in gaz.by_key
def test_a_joined_line_is_several_places(store: Store) -> None:
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
uid = posting("1").uid
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
(2, "Warsaw", "Warsaw")]
role = store.role(uid)
assert role is not None and role.place == "Copenhagen; London; Warsaw"
assert [p.name for p in role.places] == ["Copenhagen", "London", "Warsaw"]
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
store.upsert_role(posting("1", place="Berlin, Munich", places=[
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
{"text": "Remote (DE)", "remote": True}]))
assert texts(store, posting("1").uid) == [
(0, "Berlin", "Berlin"), (1, "Munich", "Munich"), (2, "Remote (DE)", "Germany")]
def test_places_without_a_line_make_the_line(store: Store) -> None:
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
{"text": "Malmö", "remote": None}]))
role = store.role(posting("1").uid)
assert role is not None
assert (role.place, role.country) == ("Lund; Malmö", "SE")
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
store.upsert_role(posting("1", place="Aarhus"))
store.upsert_role(posting("1"))
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
store.upsert_role(posting("1", place="Odense"))
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
store.upsert_role(posting("1", place="Aarhus"))
uid = posting("1").uid
with sqlite3.connect(store.path) as conn:
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
store.upsert_role(posting("1", place="Aalborg"))
assert [(p.text, p.url) for p in store.role_places(uid)] == [
("Aalborg", ""), ("Odense", "https://copy")]
def test_an_extracted_place_is_resolved(store: Store) -> None:
store.upsert_role(posting("1"))
uid = posting("1").uid
store.set_extracted(uid, {"place": "Lund, Sweden"})
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
# ---------------------------------------------------------- filter and gates
def uids(store: Store, *tokens: str) -> set[str]:
return {r.uid for r in store.roles(limit=100_000, places=list(tokens), by_score=False)}
@pytest.fixture
def several(store: Store) -> dict[str, str]:
lines = {"cph": "Copenhagen; London", "eu": "Berlin, Germany; Paris, France",
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
for key, line in lines.items():
store.upsert_role(posting(key, place=line))
return {key: posting(key).uid for key in lines}
def test_a_country_is_every_place_in_it(store: Store, several: dict[str, str]) -> None:
assert uids(store, "country:DK") == {several["cph"], several["billund"]}
assert uids(store, "country:SE") == {several["malmo"], several["lund"], several["remote"]}
assert uids(store, "country:DE") == {several["eu"]}
assert uids(store, "country:GB") == {several["cph"]}
def test_europe_is_every_european_role(store: Store, several: dict[str, str]) -> None:
assert uids(store, "country:EU") == set(several.values()) - {several["ny"], several["zeiss"]}
def test_a_region_is_every_town_in_it(store: Store, several: dict[str, str]) -> None:
assert uids(store, "region:skane") == {several["malmo"], several["lund"]}
assert uids(store, "region:oresund") == {several["malmo"], several["lund"], several["cph"]}
assert uids(store, "place:skane") == uids(store, "region:skane")
def test_a_town_is_its_place_and_lund_is_not_in_billund(store: Store,
several: dict[str, str]) -> None:
assert uids(store, "place:lund") == {several["lund"]}
assert uids(store, "place:kobenhavn") == {several["cph"]}
def test_a_town_the_gazetteer_lacks_is_found_by_its_words(store: Store,
several: dict[str, str]) -> None:
assert uids(store, "place:oberkochen") == {several["zeiss"]}
def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
with pytest.raises(ValueError):
store.roles(places=["region:atlantis"])
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
store.upsert_role(posting("1", country="DK"))
assert uids(store, "country:DK") == {posting("1").uid}
def gates(store: Store, gate: Gate) -> dict[str, int]:
agent = store.save_agent("A", gates=[gate])
return {c.uid: c.gates[0] for c in store.score_candidates(agent.id, agent.gates)}
def test_a_country_gate_is_met_by_any_place_in_the_country(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="country", values=["DK", "SE"], mode="any"))
assert {uid for uid, g in found.items() if g == 1} == {
several[k] for k in ("cph", "billund", "malmo", "lund", "remote")}
# Only a role with neither a country nor a known place has no evidence.
assert {uid for uid, g in found.items() if g == -1} == {several["zeiss"]}
assert found[several["ny"]] == 0 and found[several["eu"]] == 0
def test_the_eu_value_of_a_country_gate_is_europe(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="country", values=["EU"]))
assert found[several["eu"]] == 1 and found[several["cph"]] == 1
assert found[several["ny"]] == 0
def test_a_place_gate_matches_over_the_gazetteer(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="place", values=["Malmö", "Lund", "oberkochen"],
mode="any"))
assert {uid for uid, g in found.items() if g == 1} == {
several["malmo"], several["lund"], several["zeiss"]}
# ------------------------------------------------------------ unresolved texts
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
store.upsert_role(posting(str(n), place=line))
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
("jena", "Jena", 1)]
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
def test_an_alias_links_what_it_names(store: Store) -> None:
store.upsert_role(posting("1", place="Kbh. K"))
store.upsert_role(posting("2", place="Copenhagen"))
uid = posting("1").uid
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
assert texts(store, uid) == [(0, "Kbh. K", "Copenhagen")]
assert store.unresolved_places() == []
assert uids(store, "country:DK") == {uid, posting("2").uid}
# The second store, another process, sees the new alias too.
assert Store(store.path).gazetteer().lookup("place:kbh") == \
store.gazetteer().lookup("place:copenhagen")
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
store.upsert_role(posting("1", place="Oberkochen; Jena"))
store.upsert_role(posting("2", place="Paderborn, Germany"))
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
store.add_place("Paderborn", "city", "country:DE")
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
assert texts(store, posting("2").uid) == [(0, "Paderborn, Germany", "Paderborn")]
assert uids(store, "place:oberkochen") == {posting("1").uid}
assert store.gazetteer().country(oberkochen) == "DE"
with pytest.raises(ValueError):
store.add_place("Oberkochen", "city", "country:DE")
with pytest.raises(ValueError):
store.add_place("Atlantis", "country", "country:EU")
def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
with pytest.raises(ValueError):
store.add_place_alias("Kbh", "place:atlantis")
with pytest.raises(ValueError):
store.add_place_alias("!!", "place:copenhagen")
def test_relink_places_resolves_every_role_again(store: Store) -> None:
store.upsert_role(posting("1", place="Oberkochen"))
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE role_places SET place_id = "
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
assert store.relink_places() == 1
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "")]
# --------------------------------------------------------- against the words
#: Lines of `test_places.LINES` the word rule found for a token and the
#: gazetteer does not, each rightly: the London in Canada, and a town named
#: after Zurich in Illinois.
NOT_FOUND_ANY_MORE: dict[str, set[str]] = {
"country:GB": {"London, Canada; Remote, United States; Oak Brook, United States"},
"place:london": {"London, Canada; Remote, United States; Oak Brook, United States"},
"place:zurich": {"Lake Zurich, Illinois, United States"},
}
@pytest.fixture(scope="module")
def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tuple[str, str]]]:
store = Store(tmp_path_factory.mktemp("real") / "roledesk.db")
lines = {}
for n, (place, country) in enumerate(LINES):
item = posting(f"line{n}", place=place, country=country)
store.upsert_role(item)
lines[item.uid] = (place, country)
return store, lines
@pytest.mark.parametrize("token", ORACLE_TOKENS)
def test_the_gazetteer_finds_what_the_words_found(
real: tuple[Store, dict[str, tuple[str, str]]], token: str) -> None:
store, lines = real
by_words = {uid for uid, (place, country) in lines.items()
if places.matches(fold(place), country, token)}
by_places = uids(store, token)
lost = {lines[uid][0] for uid in by_words - by_places}
assert lost <= NOT_FOUND_ANY_MORE.get(token, set()), (token, lost)
+139 -12
View File
@@ -14,8 +14,8 @@ from desk.models import Item
import numpy as np
from app.models import (
LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key,
posted_at, posted_host, title_key,
LEVELS, Application, Gate, Limits, Prompt, company_key, place_key,
posted_at, posted_host, title_key, trusted_posted,
)
from app.storage.store import pack
from app.storage import LATEST, Store
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 18
assert LATEST == 19
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -88,8 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
def _undo_v19(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v18 shape: no gazetteer, no role places,
no teaser, one level."""
for table in ("role_places", "place_includes", "place_aliases", "places"):
conn.execute(f"DROP TABLE {table}")
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
def _undo_v18(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v17 shape: no copies."""
_undo_v19(conn)
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
@@ -166,7 +176,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18]
assert migrated.migrations_applied == [15, 16, 17, 18, 19]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -371,7 +381,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -395,7 +405,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18]
assert migrated.migrations_applied == [16, 17, 18, 19]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -424,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18]
assert migrated.migrations_applied == [17, 18, 19]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -517,7 +527,7 @@ def test_posted_host_reads_board_then_provider() -> None:
def test_posted_at_trusts_only_a_known_host() -> None:
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS
assert trusted_posted("greenhouse") and not trusted_posted("taleo")
early = "2020-01-01T00:00:00+00:00"
late = "2026-09-23T09:00:00+00:00"
# A trusted host's earlier date wins over first_seen...
@@ -527,6 +537,29 @@ def test_posted_at_trusts_only_a_known_host() -> None:
assert posted_at(early, late, "") == late
def test_a_reader_that_declares_reads_decides_its_own_date_trust(
monkeypatch: pytest.MonkeyPatch) -> None:
"""`fields.posted` of a declared reader wins over the fallback set, either
way; a reader without `READS`, or a provider that is none, keeps it."""
import app.models as models
monkeypatch.setattr(models, "_declared_posted",
lambda: {"greenhouse": "listed", "gem": "trusted"})
assert not trusted_posted("greenhouse")
assert trusted_posted("gem")
assert trusted_posted("jobindex")
assert not trusted_posted("taleo")
def test_the_declarations_are_read_off_the_readers() -> None:
import app.models as models
from plugins.readers import READERS
declared = models._declared_posted()
assert set(declared) <= set(READERS)
assert all(value in ("trusted", "listed", "none") for value in declared.values())
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
first_seen = "2026-09-23T09:00:00+00:00"
future = "2026-09-24T00:00:00+00:00" # after first_seen
@@ -574,6 +607,92 @@ def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store)
assert store.roles_to_embed("m", 10) == []
def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
store: Store) -> None:
from app.jobs.embed import document
listed = posting(teaser="Build the payments core.")
listed.body = ""
bare = posting()
bare.body = ""
store.upsert_role(bare) # the title alone
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(listed) # a teaser beats the title
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and role.teaser == "Build the payments core."
assert document(role) == "Senior Backend Engineer (m/w/d)\nBuild the payments core."
assert store.roles_page()[0].excerpt == "Build the payments core."
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(bare) # a pull without it keeps it
assert store.roles_to_embed("m", 10) == []
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and document(role).endswith("We build things.")
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
assert store.roles_to_embed("m", 10) == []
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
store.upsert_role(posting("1")) # "Senior Backend Engineer"
uid = posting("1").uid
def levels() -> tuple[str, str, str, str]:
role = store.role(uid)
assert role is not None
return (role.seniority, role.seniority_model, role.seniority_stated,
role.seniority_title)
assert levels() == ("senior", "", "", "senior")
store.upsert_role(posting("1", seniority="lead"))
assert levels() == ("lead", "", "lead", "senior")
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
assert levels() == ("senior", "", "", "senior")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="lead"))
assert levels() == ("staff", "staff", "lead", "senior")
def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> None:
path = tmp_path / "v18.db"
store = Store(path)
joined = posting("joined", place="Copenhagen; Malmö")
detailed = posting("detailed", place="Aarhus")
stated = posting("stated", seniority="lead")
extracted = posting("extracted")
for item in (joined, detailed, stated, extracted):
store.upsert_role(item)
with sqlite3.connect(path) as conn:
_undo_v19(conn)
# A role whose own page was read is pulled with an empty place in raw.
conn.execute("UPDATE roles SET raw = '{}' WHERE uid = ?", (detailed.uid,))
conn.execute("UPDATE roles SET extracted_at = '2026-09-23', seniority = 'staff' "
"WHERE uid = ?", (extracted.uid,))
conn.execute("PRAGMA user_version = 18")
migrated = Store(path)
assert migrated.migrations_applied == [19]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
with sqlite3.connect(path) as conn:
levels = {uid: rest for uid, *rest in conn.execute(
"SELECT uid, seniority, seniority_model, seniority_stated, seniority_title, teaser "
"FROM roles")}
assert levels == {
joined.uid: ["senior", "", "", "senior", ""],
detailed.uid: ["senior", "", "", "senior", ""],
stated.uid: ["lead", "", "lead", "senior", ""],
extracted.uid: ["staff", "staff", "", "senior", ""],
}
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
salary_max=95000, currency="EUR", department="Eng"))
@@ -738,12 +857,20 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None:
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
store: Store) -> None:
store.upsert_role(posting("1", seniority="junior"))
uid = posting("1").uid
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
store.set_extracted(uid, {"seniority": "senior"})
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_title,
role.seniority_model) == ("junior", "junior", "senior", "")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="junior"))
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
("staff", "junior", "staff")
def test_company_falls_back_to_the_subscription(store: Store) -> None: