Merge branch 'wp-e1-places'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
5f0f4ad5b7
10 files changed
+1437
-98
No files matched your search
+6
-2
@@ -2,7 +2,10 @@
|
||||
|
||||
The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors`
|
||||
(see `Store.put_vectors`). What gets encoded is the title and the opening of
|
||||
the description, which is what says what a role is.
|
||||
the description, which is what says what a role is; while a role has no
|
||||
description, the teaser its host's list carried, and while it has neither,
|
||||
the title alone. `Store.upsert_role` drops the vector when a better text
|
||||
arrives, so the role is encoded again.
|
||||
|
||||
sentence-transformers is optional, as in newsdesk. Without it this task puts
|
||||
one notice on screen and does nothing; `score` then leaves `fit` out and ranks
|
||||
@@ -55,7 +58,8 @@ def wants_e5_prefix(model: str) -> bool:
|
||||
|
||||
|
||||
def document(role: Role) -> str:
|
||||
return f"{role.title}\n{role.description[:BODY_CHARS]}"
|
||||
"""The title and the description, else the teaser."""
|
||||
return f"{role.title}\n{(role.description or role.teaser)[:BODY_CHARS]}"
|
||||
|
||||
|
||||
def role_text(model: str, role: Role) -> str:
|
||||
|
||||
+73
-16
@@ -12,7 +12,10 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, Literal, get_args
|
||||
|
||||
from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator
|
||||
@@ -358,8 +361,9 @@ _REGIONS = ("europe", "emea", "eu", "european union", "eea")
|
||||
#: state or a Canadian province in the same part of the line beats one of
|
||||
#: these ("London, Ontario" is Canada); a country name does not lose to it.
|
||||
_CITIES: dict[str, str] = {
|
||||
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
|
||||
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
|
||||
**dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck"), "AT"),
|
||||
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege",
|
||||
"luik"), "BE"),
|
||||
**dict.fromkeys(("sofia",), "BG"),
|
||||
**dict.fromkeys(("zagreb",), "HR"),
|
||||
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
|
||||
@@ -369,7 +373,7 @@ _CITIES: dict[str, str] = {
|
||||
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
|
||||
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
|
||||
**dict.fromkeys(("tallinn", "tartu"), "EE"),
|
||||
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
|
||||
**dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"),
|
||||
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
|
||||
"lille", "rennes", "amiens"), "FR"),
|
||||
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
|
||||
@@ -378,27 +382,30 @@ _CITIES: dict[str, str] = {
|
||||
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
|
||||
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
|
||||
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
|
||||
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
|
||||
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck"), "DE"),
|
||||
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
|
||||
**dict.fromkeys(("budapest",), "HU"),
|
||||
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
|
||||
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
|
||||
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
|
||||
"florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples",
|
||||
"napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"),
|
||||
**dict.fromkeys(("riga",), "LV"),
|
||||
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
|
||||
**dict.fromkeys(("luxembourg", "luxemburg"), "LU"),
|
||||
**dict.fromkeys(("valletta",), "MT"),
|
||||
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
||||
"delft"), "NL"),
|
||||
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
|
||||
"PL"),
|
||||
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
|
||||
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
|
||||
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"),
|
||||
**dict.fromkeys(("belgrade", "beograd"), "RS"),
|
||||
**dict.fromkeys(("bratislava",), "SK"),
|
||||
**dict.fromkeys(("ljubljana",), "SI"),
|
||||
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
|
||||
"granada", "santiago de compostela"), "ES"),
|
||||
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
|
||||
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras",
|
||||
"linkoping", "orebro", "norrkoping", "jonkoping"), "SE"),
|
||||
**dict.fromkeys(("reykjavik",), "IS"),
|
||||
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
|
||||
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
|
||||
@@ -408,7 +415,7 @@ _CITIES: dict[str, str] = {
|
||||
"slough", "southampton", "swindon", "liverpool", "redhill",
|
||||
"milton keynes"), "GB"),
|
||||
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
|
||||
"sankt gallen", "st gallen"), "CH"),
|
||||
"sankt gallen", "st gallen", "lucerne", "luzern"), "CH"),
|
||||
**dict.fromkeys(("kyiv", "kiev"), "UA"),
|
||||
**dict.fromkeys(("istanbul", "ankara"), "TR"),
|
||||
**dict.fromkeys(("tirana",), "AL"),
|
||||
@@ -659,9 +666,10 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
|
||||
|
||||
# -------------------------------------------------------------- posting dates
|
||||
|
||||
#: Hosts whose date is a creation or first-publication timestamp, safe to
|
||||
#: read as when a role was posted, checked against live boards on
|
||||
#: 2026-09-29:
|
||||
#: Hosts whose date is a creation or first-publication timestamp, for a
|
||||
#: reader that does not declare `READS` yet (see `trusted_posted`); a reader
|
||||
#: that declares it says so with `fields.posted="trusted"`. Checked against
|
||||
#: live boards on 2026-09-29:
|
||||
#:
|
||||
#: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers
|
||||
#: `first_published` over `updated_at`. A live board (adyen) showed a job
|
||||
@@ -708,14 +716,39 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
|
||||
#: HR Manager) state a date nobody checked, and Pinpoint states none.
|
||||
#:
|
||||
#: A host not listed here, or a role whose stored `published_at` fails the
|
||||
#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the
|
||||
#: safe default this migration is replacing everywhere else.
|
||||
TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({
|
||||
#: sanity check in `posted_at`, keeps `first_seen` as its posted date.
|
||||
_UNDECLARED_TRUSTED: frozenset[str] = frozenset({
|
||||
"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
|
||||
"personio", "teamtailor", "jobindex", "adzuna", "platsbanken",
|
||||
"workday", "successfactors", "oracle", "phenom", "bamboohr", "jobylon", "rippling",
|
||||
})
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _declared_posted() -> dict[str, str]:
|
||||
"""`fields.posted` of every reader that declares `READS`, by name. Read
|
||||
once: the readers are loaded when `plugins.readers` is imported. `plugins`
|
||||
is a top-level folder beside `app`, reached as `app/jobs/extract.py`
|
||||
reaches it."""
|
||||
root = str(Path(__file__).resolve().parents[1])
|
||||
if root not in sys.path:
|
||||
sys.path.insert(0, root)
|
||||
from plugins.readers import READERS
|
||||
|
||||
return {name: module.READS.fields.posted for name, module in READERS.items()
|
||||
if getattr(module, "READS", None) is not None}
|
||||
|
||||
|
||||
def trusted_posted(host: str) -> bool:
|
||||
"""Whether `host` states a date that is safe to read as when a role was
|
||||
posted: its reader declares `fields.posted="trusted"`, or, for a reader
|
||||
without `READS` (or a search provider not yet a reader), it is in
|
||||
`_UNDECLARED_TRUSTED`."""
|
||||
declared = _declared_posted()
|
||||
if host in declared:
|
||||
return declared[host] == "trusted"
|
||||
return host in _UNDECLARED_TRUSTED
|
||||
|
||||
|
||||
#: A `published_at` before this is a parsing slip or a placeholder, not a
|
||||
#: real posting date: no board here existed before 2015.
|
||||
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
||||
@@ -732,7 +765,7 @@ def posted_host(raw: dict[str, Any]) -> str:
|
||||
def posted_at(published_at: str, first_seen: str, host: str) -> str:
|
||||
"""The date a role counts as posted: the earlier of `first_seen` (when
|
||||
the app itself first stored it) and `published_at` (what the board
|
||||
claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and
|
||||
claims), but only when `trusted_posted(host)` and
|
||||
`published_at` is neither absurdly old nor later than `first_seen` — a
|
||||
role cannot be posted after the app already saw it, trusted host or not,
|
||||
which also catches a `published_at` the board reports in the future
|
||||
@@ -740,7 +773,7 @@ def posted_at(published_at: str, first_seen: str, host: str) -> str:
|
||||
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
|
||||
is exact and needs no parsing. A role with no trusted date, or with one
|
||||
that fails the check, keeps `first_seen`."""
|
||||
if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen:
|
||||
if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen:
|
||||
return published_at
|
||||
return first_seen
|
||||
|
||||
@@ -775,6 +808,20 @@ class RoleCopy(Model):
|
||||
posted_at: str = ""
|
||||
|
||||
|
||||
class RolePlace(Model):
|
||||
"""One place of a role (`role_places`): the host's `text`, and the place
|
||||
of the gazetteer it resolves to, `place_id` None (and `name` and `kind`
|
||||
empty) while it resolves to none. `url` is the link of a copy folded into
|
||||
the role, '' for the role's own places."""
|
||||
|
||||
position: int = 0
|
||||
text: str = ""
|
||||
place_id: int | None = None
|
||||
name: str = ""
|
||||
kind: str = ""
|
||||
url: str = ""
|
||||
|
||||
|
||||
class Role(Model):
|
||||
"""A row of `roles`: one posting on one board."""
|
||||
|
||||
@@ -797,13 +844,20 @@ class Role(Model):
|
||||
#: could be normalised into `salary_min`/`salary_max`/`currency`
|
||||
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
|
||||
salary_text: str = ""
|
||||
#: The level shown and filtered on: the first filled of `seniority_model`,
|
||||
#: `seniority_stated` and `seniority_title`.
|
||||
seniority: str = ""
|
||||
seniority_model: str = ""
|
||||
seniority_stated: str = ""
|
||||
seniority_title: str = ""
|
||||
function: str = ""
|
||||
permit: Permit = ""
|
||||
#: ISO 639-1 codes the posting says are required, lowercase.
|
||||
languages: JsonStrings = []
|
||||
description: str = ""
|
||||
feed_text: str = ""
|
||||
#: The short text the host's list carries, until `description` arrives.
|
||||
teaser: str = ""
|
||||
original_url: str = ""
|
||||
published_at: str
|
||||
#: The date this role counts as posted: see `posted_at`. Used for
|
||||
@@ -838,6 +892,9 @@ class Role(Model):
|
||||
#: The open copies of this role, oldest first, when it is the kept one;
|
||||
#: empty otherwise. Not a column: filled by `Store.role`.
|
||||
copies: list[RoleCopy] = []
|
||||
#: Every place of the role, in the host's order. Not a column: filled by
|
||||
#: `Store.role` from `role_places`.
|
||||
places: list[RolePlace] = []
|
||||
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
|
||||
+313
-5
@@ -19,8 +19,13 @@ several hundred milliseconds a query, as a set intersection on the words of a
|
||||
line it costs a few. `register` gives a connection that answer as the SQL
|
||||
function `place_match`, and `match_sql` returns the clause that calls it.
|
||||
`suggest` turns typed text into tokens for the autocomplete; `label` reads a
|
||||
chip's text back from the address. This module opens no database; the caller
|
||||
hands over the places it holds, or its connection.
|
||||
chip's text back from the address.
|
||||
|
||||
The filter and the gates no longer match words: a role's places are resolved
|
||||
against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
|
||||
and a token is the places under the place it names. `matches` stays the rule
|
||||
for a `place:` token the gazetteer does not know. This module opens no
|
||||
database; the caller hands over the places it holds, or its connection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -32,10 +37,12 @@ from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
|
||||
from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold
|
||||
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
|
||||
country_of)
|
||||
|
||||
__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known",
|
||||
"label", "match_sql", "matches", "parse", "register", "suggest", "towns"]
|
||||
__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed",
|
||||
"Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse",
|
||||
"placeless", "register", "seed", "suggest", "towns"]
|
||||
|
||||
|
||||
def fold(text: str) -> str:
|
||||
@@ -538,6 +545,9 @@ _PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE)
|
||||
#: Danish boards append "and the possibility of working from home".
|
||||
_AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE)
|
||||
|
||||
#: A slash between two places, "Malmö/Lund", "Berlin / Munich".
|
||||
_SLASH = re.compile(r"\s*/\s*")
|
||||
|
||||
#: A postal district after the town, "København K", "Aarhus C", "Odense SV".
|
||||
_DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
|
||||
|
||||
@@ -665,3 +675,301 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
|
||||
regions.sort(key=lambda s: (-s.count, s.label))
|
||||
towns.sort(key=lambda s: (-s.count, s.label))
|
||||
return (countries + regions + towns)[:limit]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ gazetteer
|
||||
|
||||
#: The kinds of place, widest first: the index is how specific a kind is.
|
||||
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
|
||||
|
||||
#: Entries of `_CITIES` that name a region rather than a town.
|
||||
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
|
||||
"bay area"))
|
||||
|
||||
#: Words a place text holds when it names how or where-ever a role is worked
|
||||
#: rather than a place: "Remote", "Multiple Locations", "Home based". A text
|
||||
#: of these alone is no place, so it is never listed as unresolved.
|
||||
_PLACELESS = frozenset((
|
||||
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
|
||||
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
|
||||
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
|
||||
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
|
||||
|
||||
|
||||
def placeless(text: str) -> bool:
|
||||
"""Whether a place text names no place at all: empty, or only numbers and
|
||||
words such as `Remote`, `Hybrid`, `2 Locations`."""
|
||||
return all(word in _PLACELESS or word.isdigit() for word in fold(text).split())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Seed:
|
||||
"""One place as the code knows it: `key` is unique within `kind` (an ISO
|
||||
code for a country, a region's key, a town's group), `parent` the
|
||||
`(kind, key)` of the place it lies in, `aliases` the folds it is written
|
||||
as."""
|
||||
|
||||
kind: str
|
||||
key: str
|
||||
name: str
|
||||
parent: tuple[str, str] | None
|
||||
aliases: tuple[str, ...]
|
||||
|
||||
|
||||
def _subdivisions() -> list[tuple[str, str, str]]:
|
||||
"""`(country, code, name)` of every US state and Canadian province."""
|
||||
found = []
|
||||
for line in _SUBDIVISIONS.split("\n"):
|
||||
if line:
|
||||
country, rest = line.split(" ", 1)
|
||||
for entry in rest.split("|"):
|
||||
name, code = entry.strip().rsplit(" ", 1)
|
||||
found.append((country, code, name))
|
||||
return found
|
||||
|
||||
|
||||
def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
|
||||
"""The gazetteer the dictionaries in code describe, parents before their
|
||||
children, and the inclusions beyond the tree (Øresund holds the Capital
|
||||
Region and Skåne, which lie in two countries).
|
||||
|
||||
Europe is a continent over the countries of `_EUROPE`, and is also what
|
||||
`EU`, `EMEA` and `European Union` name. Every country of `_COUNTRY_NAMES`
|
||||
by its names; the US states and Canadian provinces as regions; the
|
||||
regions of `REGIONS`; the towns of the alias groups, of the regions and
|
||||
of `_CITIES`, one place per group (`Copenhagen`, `København`,
|
||||
`Kopenhamn` are one), in their region when a region lists them and else
|
||||
in their country."""
|
||||
europe = ("continent", "europe")
|
||||
seeds: dict[tuple[str, str], Seed] = {}
|
||||
|
||||
def add(kind: str, key: str, name: str, parent: tuple[str, str] | None,
|
||||
aliases: Iterable[str]) -> None:
|
||||
folds = tuple(dict.fromkeys(a for a in aliases if a))
|
||||
if (kind, key) in seeds:
|
||||
old = seeds[(kind, key)]
|
||||
folds = tuple(dict.fromkeys((*old.aliases, *folds)))
|
||||
seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds)
|
||||
else:
|
||||
seeds[(kind, key)] = Seed(kind, key, name, parent, folds)
|
||||
|
||||
add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS)))
|
||||
for code, names in _COUNTRY_LABELS.items():
|
||||
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
|
||||
for country, code, name in _subdivisions():
|
||||
add("region", f"{country}-{code}", name, ("country", country), (fold(name),))
|
||||
for region in REGIONS.values():
|
||||
parent = ("country", region.country) if region.country else europe
|
||||
add("region", region.key, region.label, parent,
|
||||
(v for name in (region.key, region.label, *region.names)
|
||||
for v in _variants(fold(name))))
|
||||
for name in sorted(_CITY_REGIONS):
|
||||
add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,))
|
||||
|
||||
# Towns, one per group, in the region that lists them first.
|
||||
town_region: dict[str, str] = {}
|
||||
for region in REGIONS.values():
|
||||
for name in region.towns:
|
||||
town_region.setdefault(_group(fold(name)), region.key)
|
||||
folds: dict[str, dict[str, None]] = {}
|
||||
shown: dict[str, str] = {}
|
||||
|
||||
def add_town(folded: str, name: str) -> None:
|
||||
key = _group(folded)
|
||||
folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded)))
|
||||
shown.setdefault(key, name)
|
||||
|
||||
for spellings in _ALIAS_GROUPS:
|
||||
for spelling in spellings:
|
||||
add_town(fold(spelling), spellings[0])
|
||||
for region in REGIONS.values():
|
||||
for name in region.towns:
|
||||
add_town(fold(name), name)
|
||||
for city in _CITIES:
|
||||
if city not in _CITY_REGIONS and city not in _REGION_BY_NAME:
|
||||
add_town(city, _TOWN_LABELS.get(city) or city.title())
|
||||
for key, forms in folds.items():
|
||||
country = next((_CITIES[f] for f in forms if f in _CITIES), "")
|
||||
if key in town_region:
|
||||
parent = ("region", town_region[key])
|
||||
elif country:
|
||||
parent = ("country", country)
|
||||
else:
|
||||
raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES")
|
||||
add("city", key, shown[key], parent, forms)
|
||||
|
||||
includes = [(("region", region.key), ("region", part))
|
||||
for region in REGIONS.values() for part in region.includes]
|
||||
return list(seeds.values()), includes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Place:
|
||||
"""A row of `places`."""
|
||||
|
||||
id: int
|
||||
kind: str
|
||||
key: str
|
||||
name: str
|
||||
parent_id: int | None
|
||||
|
||||
|
||||
class Gazetteer:
|
||||
"""The places, their aliases and what lies in what, as the database holds
|
||||
them, read into memory: what resolves a role's place text to places and a
|
||||
filter token to the places under it. Built by the store from its tables,
|
||||
and rebuilt when they change."""
|
||||
|
||||
def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]],
|
||||
aliases: Iterable[tuple[str, int]],
|
||||
includes: Iterable[tuple[int, int]] = ()) -> None:
|
||||
self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows}
|
||||
self.by_key: dict[tuple[str, str], int] = {
|
||||
(p.kind, p.key): p.id for p in self.places.values()}
|
||||
by_fold: dict[str, list[int]] = {}
|
||||
for folded, place_id in aliases:
|
||||
by_fold.setdefault(folded, []).append(place_id)
|
||||
self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()}
|
||||
self._longest = max((len(f.split()) for f in self.aliases), default=1)
|
||||
self._up: dict[int, set[int]] = {}
|
||||
self._down: dict[int, set[int]] = {}
|
||||
for p in self.places.values():
|
||||
if p.parent_id is not None:
|
||||
self._link(p.parent_id, p.id)
|
||||
for whole, part in includes:
|
||||
self._link(whole, part)
|
||||
self._ancestors: dict[int, frozenset[int]] = {}
|
||||
self._resolved: dict[str, tuple[int, ...]] = {}
|
||||
|
||||
def _link(self, whole: int, part: int) -> None:
|
||||
self._up.setdefault(part, set()).add(whole)
|
||||
self._down.setdefault(whole, set()).add(part)
|
||||
|
||||
def ancestors(self, place_id: int) -> frozenset[int]:
|
||||
"""Every place `place_id` lies in, through parents and inclusions."""
|
||||
if (found := self._ancestors.get(place_id)) is None:
|
||||
seen: set[int] = set()
|
||||
todo = list(self._up.get(place_id, ()))
|
||||
while todo:
|
||||
up = todo.pop()
|
||||
if up not in seen:
|
||||
seen.add(up)
|
||||
todo += self._up.get(up, ())
|
||||
found = self._ancestors[place_id] = frozenset(seen)
|
||||
return found
|
||||
|
||||
def under(self, ids: Iterable[int]) -> frozenset[int]:
|
||||
"""The places and every place that lies in one of them."""
|
||||
seen: set[int] = set()
|
||||
todo = list(ids)
|
||||
while todo:
|
||||
place_id = todo.pop()
|
||||
if place_id not in seen:
|
||||
seen.add(place_id)
|
||||
todo += self._down.get(place_id, ())
|
||||
return frozenset(seen)
|
||||
|
||||
def country(self, place_id: int) -> str:
|
||||
"""The ISO code of the country a place is in, '' above or outside one."""
|
||||
place: Place | None = self.places.get(place_id)
|
||||
while place is not None and place.kind != "country":
|
||||
place = self.places.get(place.parent_id) if place.parent_id is not None else None
|
||||
return place.key if place is not None else ""
|
||||
|
||||
def codes(self, ids: Iterable[int]) -> frozenset[str]:
|
||||
"""The ISO codes of the countries among `ids`, and `EU` with Europe:
|
||||
what a `roles.country` column holds for a role of one of them."""
|
||||
found = set()
|
||||
for place_id in ids:
|
||||
place = self.places[place_id]
|
||||
if place.kind == "country":
|
||||
found.add(place.key)
|
||||
elif (place.kind, place.key) == ("continent", "europe"):
|
||||
found.add("EU")
|
||||
return frozenset(found)
|
||||
|
||||
def _specific(self, place_id: int) -> int:
|
||||
return KINDS.index(self.places[place_id].kind)
|
||||
|
||||
def _hits(self, folded: str) -> list[tuple[int, ...]]:
|
||||
words = folded.split()
|
||||
hits = []
|
||||
i = 0
|
||||
while i < len(words):
|
||||
for n in range(min(self._longest, len(words) - i), 0, -1):
|
||||
if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None:
|
||||
hits.append(ids)
|
||||
i += n
|
||||
break
|
||||
else:
|
||||
i += 1
|
||||
return hits
|
||||
|
||||
def resolve(self, text: str) -> tuple[int, ...]:
|
||||
"""The places one place text names, most specific first from the left:
|
||||
`Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund`
|
||||
is Malmö and Lund, `Paderborn, Germany` is Germany, `Remote` is none.
|
||||
|
||||
Every alias the text holds as whole words is a candidate, longest
|
||||
first. A fold that names several places is the one in the country
|
||||
`country_of` reads off the text, if one is, and else the most specific
|
||||
(`Luxembourg` the town before the country). A candidate that another
|
||||
one lies in is dropped, and so is one outside the one country the text
|
||||
names, or, naming none, outside the regions it names in one country:
|
||||
`London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the
|
||||
Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several
|
||||
countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country
|
||||
`country_of` reads (from a state code, `Paris, TX`, or a code in
|
||||
parentheses, `Remote (DE)`), the text is that country."""
|
||||
folded = fold(text)
|
||||
if (found := self._resolved.get(folded)) is None:
|
||||
parts = _SLASH.split(_AND_HOME.sub("", text or ""))
|
||||
found = tuple(dict.fromkeys(
|
||||
p for part in parts for sub in _PART.split(part) for p in self._resolve(sub)))
|
||||
self._resolved[folded] = found
|
||||
return found
|
||||
|
||||
def _resolve(self, text: str) -> list[int]:
|
||||
"""`resolve` for one part of a text, cut where `or`, `/` or `;` end one
|
||||
place."""
|
||||
code = country_of(text)
|
||||
country = self.by_key.get(("country", code)) if code and code != "EU" else None
|
||||
within = self.under((country,)) if country is not None else frozenset()
|
||||
picked: list[int] = []
|
||||
for hit in self._hits(fold(text)):
|
||||
choice = [p for p in hit if p in within] or list(hit)
|
||||
best = max(choice, key=self._specific)
|
||||
if best not in picked:
|
||||
picked.append(best)
|
||||
# The widest units named hold the rest: one country, else the regions
|
||||
# of one country. Several countries are a list, not a path.
|
||||
for kind in ("country", "region"):
|
||||
named = [p for p in picked if self.places[p].kind == kind]
|
||||
if len({self.country(p) for p in named}) == 1:
|
||||
holds = self.under(named)
|
||||
picked = [p for p in picked if p in holds or self.places[p].kind == "continent"
|
||||
or any(p in self.ancestors(q) for q in named)]
|
||||
if named:
|
||||
break
|
||||
wider = {a for p in picked for a in self.ancestors(p)}
|
||||
picked = [p for p in picked if p not in wider]
|
||||
if country is not None and not any(p in within or p in self.ancestors(country)
|
||||
for p in picked):
|
||||
picked = [country]
|
||||
return picked
|
||||
|
||||
def lookup(self, token: str) -> frozenset[int] | None:
|
||||
"""The places a filter token names, before what lies in them: `country:DK`
|
||||
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
|
||||
every place written so (a town, or a region or a country by name).
|
||||
None when the gazetteer knows no such place."""
|
||||
tok = parse(token)
|
||||
if tok.kind == "country":
|
||||
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
|
||||
found = {self.by_key[key]} if key in self.by_key else set()
|
||||
elif tok.kind == "region":
|
||||
found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \
|
||||
else set(self.aliases.get(tok.value, ()))
|
||||
else:
|
||||
found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())}
|
||||
return frozenset(found) or None
|
||||
@@ -3,6 +3,8 @@
|
||||
schema.py the tables, and the numbered migrations that bring an older
|
||||
file to the current shape
|
||||
store.py `Store`, every SQL statement in the application
|
||||
gazetteer.py the places tables: seeded from the code, read back, and a
|
||||
role's places written and relinked
|
||||
|
||||
Written for roledesk rather than imported from newsdesk: `desk` has no storage
|
||||
package on purpose, and the connection plumbing here is a deliberate copy.
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""The gazetteer's tables: seeding them from the code, reading them back, and
|
||||
writing a role's places.
|
||||
|
||||
`places`, `place_aliases` and `place_includes` hold the places
|
||||
(`app.places.seed` on a new or migrated file, and what a person adds later).
|
||||
`role_places` holds one row per place of a role: the host's own text, its
|
||||
fold, and the place it resolves to, NULL for a text the gazetteer does not
|
||||
know. A text that names several places ("Malmö/Lund") is one row per place,
|
||||
each with the same `position` and `text`. A row with a `url` is the place of
|
||||
a copy folded into the role (`app/dupes.py`) and is not the role's own: only
|
||||
own rows (`url = ''`) are rewritten here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
from app import places
|
||||
|
||||
#: What changes when the gazetteer does: rows are only ever added.
|
||||
VERSION = ("SELECT (SELECT MAX(rowid) FROM places), (SELECT MAX(rowid) FROM place_aliases), "
|
||||
"(SELECT COUNT(*) FROM place_includes)")
|
||||
|
||||
|
||||
def seed(conn: sqlite3.Connection) -> int:
|
||||
"""Insert what `places.seed` describes, leaving rows already there alone.
|
||||
Returns how many places were added."""
|
||||
seeds, includes = places.seed()
|
||||
ids = {(kind, key): place_id for place_id, kind, key in
|
||||
conn.execute("SELECT id, kind, key FROM places")}
|
||||
added = 0
|
||||
for s in seeds:
|
||||
if (s.kind, s.key) not in ids:
|
||||
parent = ids[s.parent] if s.parent else None
|
||||
ids[(s.kind, s.key)] = conn.execute(
|
||||
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
|
||||
(s.kind, s.key, s.name, parent)).lastrowid
|
||||
added += 1
|
||||
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
||||
[(a, ids[(s.kind, s.key)]) for a in s.aliases])
|
||||
conn.executemany("INSERT OR IGNORE INTO place_includes (place_id, part_id) VALUES (?, ?)",
|
||||
[(ids[whole], ids[part]) for whole, part in includes])
|
||||
return added
|
||||
|
||||
|
||||
def load(conn: sqlite3.Connection) -> places.Gazetteer:
|
||||
return places.Gazetteer(
|
||||
[tuple(r) for r in conn.execute("SELECT id, kind, key, name, parent_id FROM places")],
|
||||
[tuple(r) for r in conn.execute("SELECT fold, place_id FROM place_aliases")],
|
||||
[tuple(r) for r in conn.execute("SELECT place_id, part_id FROM place_includes")])
|
||||
|
||||
|
||||
def texts(raw: Mapping[str, Any]) -> list[str]:
|
||||
"""A role's place texts from `Item.raw`: the `text` of each of `places`
|
||||
(`[{"text": ..., "remote": ...}]`, the reader contract), else `place` split
|
||||
at `;`, which is how the older readers join several places into one line.
|
||||
Blank and repeated texts are dropped."""
|
||||
given = raw.get("places")
|
||||
if isinstance(given, list) and given:
|
||||
found = [str((p.get("text") if isinstance(p, Mapping) else p) or "") for p in given]
|
||||
else:
|
||||
found = str(raw.get("place") or "").split(";")
|
||||
return list(dict.fromkeys(t.strip() for t in found if t and t.strip()))
|
||||
|
||||
|
||||
def rows(gaz: places.Gazetteer, uid: str, place_texts: Sequence[str]
|
||||
) -> list[tuple[str, int, str, str, int | None]]:
|
||||
"""`(uid, position, text, fold, place_id)` for each place of the texts."""
|
||||
found: list[tuple[str, int, str, str, int | None]] = []
|
||||
for position, text in enumerate(place_texts):
|
||||
folded = places.fold(text)
|
||||
found += [(uid, position, text, folded, place_id)
|
||||
for place_id in (gaz.resolve(text) or (None,))]
|
||||
return found
|
||||
|
||||
|
||||
_INSERT = "INSERT INTO role_places (uid, position, text, fold, place_id) VALUES (?, ?, ?, ?, ?)"
|
||||
|
||||
|
||||
def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
|
||||
place_texts: Sequence[str]) -> None:
|
||||
"""Replace one role's own places with these texts, resolved."""
|
||||
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
|
||||
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
||||
|
||||
|
||||
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
||||
"""`write` for many roles in one statement each way. Returns rows written."""
|
||||
batch = list(by_uid)
|
||||
conn.executemany("DELETE FROM role_places WHERE uid = ? AND url = ''",
|
||||
[(uid,) for uid, _ in batch])
|
||||
new = [row for uid, found in batch for row in rows(gaz, uid, found)]
|
||||
conn.executemany(_INSERT, new)
|
||||
return len(new)
|
||||
|
||||
|
||||
def relink(conn: sqlite3.Connection, gaz: places.Gazetteer, words: str | None = None) -> int:
|
||||
"""Resolve the own places of every role again, or only of the roles with a
|
||||
place text that holds `words` (a fold) as whole words, after an alias or a
|
||||
place was added. Returns how many roles now resolve differently."""
|
||||
sql = "SELECT uid, position, text, place_id FROM role_places WHERE url = ''"
|
||||
params: list[Any] = []
|
||||
if words:
|
||||
sql += " AND ' ' || fold || ' ' LIKE ? ESCAPE '\\'"
|
||||
escaped = words.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
params.append(f"% {escaped} %")
|
||||
sql = ("SELECT uid, position, text, place_id FROM role_places WHERE url = '' AND uid IN "
|
||||
f"(SELECT uid FROM ({sql}))")
|
||||
before: dict[str, dict[tuple[int, str], set[int | None]]] = {}
|
||||
for uid, position, text, place_id in conn.execute(sql + " ORDER BY uid, position", params):
|
||||
before.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
|
||||
changed = []
|
||||
for uid, held in before.items():
|
||||
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in held}
|
||||
if now != held:
|
||||
changed.append((uid, [text for _, text in sorted(held)]))
|
||||
write_many(conn, gaz, changed)
|
||||
return len(changed)
|
||||
+156
-7
@@ -18,8 +18,9 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import country_of, posted_at, posted_host
|
||||
from app.models import SENIORITIES, country_of, posted_at, posted_host
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -44,8 +45,9 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- require the country: "UK | Remote" is not remote from Berlin.
|
||||
place TEXT NOT NULL DEFAULT '',
|
||||
place_key TEXT NOT NULL DEFAULT '',
|
||||
-- places.fold(place): what the Place filter and the place gate match
|
||||
-- whole words against, through the place_match function (app/places.py).
|
||||
-- places.fold(place): what a place: token the gazetteer does not know is
|
||||
-- matched against as whole words, through place_match (app/places.py).
|
||||
-- The filter and the gates otherwise read role_places.
|
||||
place_fold TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
@@ -53,7 +55,8 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
-- seniority.from_title(title) at ingest, until extract reads the posting.
|
||||
-- The level shown and filtered on: the first filled of seniority_model,
|
||||
-- seniority_stated and seniority_title (the last columns), written with them.
|
||||
seniority TEXT NOT NULL DEFAULT '',
|
||||
function TEXT NOT NULL DEFAULT '',
|
||||
-- Whether the posting says it sponsors a work permit, and the languages
|
||||
@@ -69,7 +72,7 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- stored the role; posted_at (app.models.posted_at) is the earlier of
|
||||
-- first_seen and published_at, trusted only for a host known to report a
|
||||
-- creation date rather than one that moves on an edit
|
||||
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
|
||||
-- (app.models.trusted_posted) — recency and the feed's newest order
|
||||
-- read posted_at, first_seen stays "when the app found it". closed_at is
|
||||
-- set when a board fetched in full no longer lists the role, and cleared
|
||||
-- if it returns.
|
||||
@@ -100,7 +103,62 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- precedes, which the migration tests do.
|
||||
dup_of TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
salary_text TEXT NOT NULL DEFAULT ''
|
||||
salary_text TEXT NOT NULL DEFAULT '',
|
||||
teaser TEXT NOT NULL DEFAULT '',
|
||||
seniority_model TEXT NOT NULL DEFAULT '',
|
||||
seniority_stated TEXT NOT NULL DEFAULT '',
|
||||
seniority_title TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
-- roles.teaser is the short text a list carries, kept until the full text
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
-- three levels behind roles.seniority: the model's (extract), the host's own
|
||||
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
|
||||
-- among the last columns: SQLite cannot drop a column a comment precedes,
|
||||
-- which the migration tests do.
|
||||
|
||||
-- The gazetteer: a place is a continent, a country (key the ISO code), a
|
||||
-- region or a city, in the place parent_id names. key is unique per kind:
|
||||
-- a region's key (app.places.REGIONS), a town's group. Seeded from the
|
||||
-- dictionaries in app/models.py and app/places.py (app.places.seed).
|
||||
CREATE TABLE IF NOT EXISTS places (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
|
||||
key TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
parent_id INTEGER REFERENCES places(id),
|
||||
UNIQUE (kind, key)
|
||||
);
|
||||
|
||||
-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn
|
||||
-- and copenhagen are Copenhagen. One fold may name several places
|
||||
-- (Luxembourg the town and the country).
|
||||
CREATE TABLE IF NOT EXISTS place_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
UNIQUE (fold, place_id)
|
||||
);
|
||||
|
||||
-- What a place holds beyond its children: Øresund holds the Capital Region
|
||||
-- of Denmark and Skåne, which lie in two countries.
|
||||
CREATE TABLE IF NOT EXISTS place_includes (
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (place_id, part_id)
|
||||
);
|
||||
|
||||
-- One row per place of a role (app/storage/gazetteer.py): the host's text in
|
||||
-- the host's order (position), its fold, and the place it resolves to, NULL
|
||||
-- while the gazetteer does not know it. A text naming several places is a
|
||||
-- row for each. url is the link of a copy folded into this role, '' for
|
||||
-- the role's own places.
|
||||
CREATE TABLE IF NOT EXISTS role_places (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER REFERENCES places(id),
|
||||
url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- The open vocabulary summarise reads off a posting: the role family, the
|
||||
@@ -303,6 +361,11 @@ CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
|
||||
-- Partial: all but about 3% of roles are no copy.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
ON roles(dup_of) WHERE dup_of IS NOT NULL;
|
||||
-- A role's places, and the roles in a set of places (the Place filter and
|
||||
-- the country and place gates, `uid IN (... WHERE place_id IN ...)`).
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -469,6 +532,10 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
# A Python step: the first rebuild runs the rule in app/dupes.py.
|
||||
(18, "roles.dup_of: one posting listed several times is one role, the others "
|
||||
"copies of it", lambda conn: _add_dup_of(conn)),
|
||||
# A Python step: the seed and the resolver are Python.
|
||||
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
|
||||
"places, resolved against a gazetteer; roles.teaser; the three seniority "
|
||||
"sources", lambda conn: _add_places(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -592,6 +659,81 @@ def _add_dup_of(conn: sqlite3.Connection) -> None:
|
||||
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
|
||||
|
||||
|
||||
_V19 = """
|
||||
ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT '';
|
||||
CREATE TABLE places (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
|
||||
key TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
parent_id INTEGER REFERENCES places(id),
|
||||
UNIQUE (kind, key)
|
||||
);
|
||||
CREATE TABLE place_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
UNIQUE (fold, place_id)
|
||||
);
|
||||
CREATE TABLE place_includes (
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (place_id, part_id)
|
||||
);
|
||||
CREATE TABLE role_places (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
fold TEXT NOT NULL,
|
||||
place_id INTEGER REFERENCES places(id),
|
||||
url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def _add_places(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 19's step. The tables and columns, the gazetteer seeded, and
|
||||
every role's places: the `places` of its `raw` where a reader stated
|
||||
them, else its `place` column cut at `;` (not `raw`'s, which a pull that
|
||||
read the role's own page sends empty), each resolved once per distinct
|
||||
text.
|
||||
|
||||
The level splits into its sources without changing: on a role extract
|
||||
has read, `seniority` is the model's; on any other, it is the stated
|
||||
level when `raw` states one in the vocabulary, else the title's."""
|
||||
for statement in _statements(_V19):
|
||||
conn.execute(statement)
|
||||
gazetteer.seed(conn)
|
||||
gaz = gazetteer.load(conn)
|
||||
rows = conn.execute(
|
||||
"SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall()
|
||||
by_uid: list[tuple[str, list[str]]] = []
|
||||
levels = []
|
||||
for uid, raw, place, title, level, extracted_at in rows:
|
||||
try:
|
||||
given = json.loads(raw or "{}")
|
||||
except ValueError:
|
||||
given = {}
|
||||
given = given if isinstance(given, dict) else {}
|
||||
if found := gazetteer.texts({"places": given.get("places"), "place": place}):
|
||||
by_uid.append((uid, found))
|
||||
stated = str(given.get("seniority") or "")
|
||||
stated = stated if stated in SENIORITIES else ""
|
||||
if extracted_at is not None:
|
||||
levels.append((level, stated, from_title(title) if level else "", uid))
|
||||
elif stated:
|
||||
levels.append(("", stated, from_title(title), uid))
|
||||
elif level:
|
||||
levels.append(("", "", level, uid))
|
||||
written = gazetteer.write_many(conn, gaz, by_uid)
|
||||
conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, "
|
||||
"seniority_title = ? WHERE uid = ?", levels)
|
||||
log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d",
|
||||
len(gaz.places), written, len(by_uid), len(levels))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
@@ -659,6 +801,13 @@ def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def _create(conn: sqlite3.Connection) -> None:
|
||||
"""A new file: `SCHEMA`, and the gazetteer seeded."""
|
||||
for statement in _statements(SCHEMA):
|
||||
conn.execute(statement)
|
||||
gazetteer.seed(conn)
|
||||
|
||||
|
||||
def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
||||
fresh = conn.execute(
|
||||
@@ -668,7 +817,7 @@ def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
if fresh:
|
||||
# Atomic like a migration: a file half created would otherwise carry
|
||||
# the marker table and be taken for one to migrate.
|
||||
_apply(conn, LATEST, SCHEMA)
|
||||
_apply(conn, LATEST, _create)
|
||||
else:
|
||||
applied = migrate(conn)
|
||||
for statement in _statements(INDEXES):
|
||||
|
||||
+225
-55
@@ -22,14 +22,14 @@ import numpy as np
|
||||
from desk.models import Item
|
||||
from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
|
||||
ScoredRole,
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
||||
RoleRow, ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import schema
|
||||
from app.storage import gazetteer, schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
#: empty value never overwrites a filled column, so an upsert from a board that
|
||||
@@ -101,7 +101,8 @@ _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
||||
#: one, else the description's opening. A `SUBSTR` of the source column, not
|
||||
#: the column itself, so a big `description` never leaves SQLite whole for a
|
||||
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
|
||||
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
|
||||
_EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), "
|
||||
"roles.teaser)")
|
||||
|
||||
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
|
||||
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
|
||||
@@ -177,23 +178,34 @@ def unpack(blob: bytes) -> np.ndarray:
|
||||
return np.frombuffer(blob, dtype=_DTYPE)
|
||||
|
||||
|
||||
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
||||
#: A role with no place the gazetteer resolved, over `roles r`.
|
||||
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
|
||||
|
||||
|
||||
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
"""One gate as `(empty, match, params)`: SQL that is true when the role's
|
||||
column is empty, and SQL that is true when it matches, over `roles r`.
|
||||
`params` belong to `match`; `empty` takes none."""
|
||||
`params` belong to `match`; `empty` takes none.
|
||||
|
||||
`country` and `place` match over the gazetteer, as the feed's Place
|
||||
filter does (`_places_sql`): a country gate is met by a role whose
|
||||
`country` is one of its codes or that has a place in one of them, so
|
||||
`DK` is met by `Copenhagen; London`. Its column is empty only when the
|
||||
role has neither. A place gate's value is a place token, bare text a
|
||||
`place:`."""
|
||||
column = f"r.{gate.field}"
|
||||
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
||||
marks = ", ".join("?" for _ in values) or "NULL"
|
||||
if gate.field == "salary_min":
|
||||
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
||||
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
||||
if gate.field == "country":
|
||||
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
|
||||
codes=[v.upper() for v in values])
|
||||
return f"(r.country = '' AND {_UNPLACED})", match, params
|
||||
if gate.field == "place":
|
||||
# Whole folded words, as the feed's Place filter: `lund` is not in
|
||||
# `Billund`. A value is a place token, bare text a `place:`.
|
||||
clauses = [places.match_sql(v, place="r.place_fold", country="r.country")
|
||||
for v in values]
|
||||
match = " OR ".join(sql for sql, _ in clauses) or "0"
|
||||
return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps]
|
||||
match, params = _places_sql(values, gaz, table="r")
|
||||
return "r.place = ''", match, params
|
||||
if gate.field == "company":
|
||||
keys = [company_key(v) for v in values]
|
||||
return ("r.company_key = ''",
|
||||
@@ -205,18 +217,54 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||
|
||||
|
||||
def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]:
|
||||
"""The Place filter over `roles`: any of the tokens, each a `place_match`
|
||||
call (`places.match_sql`). ValueError for a token that names nothing,
|
||||
which would otherwise match no role without saying why."""
|
||||
clauses, params = [], []
|
||||
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
|
||||
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
|
||||
"""Any of the place tokens over `table`, as a join over the gazetteer: a
|
||||
role with a place under one of the places the tokens name (`country:DK`
|
||||
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
|
||||
Malmö by any spelling), or whose `country` is one of their countries (a
|
||||
role whose board gave a country and no place). `codes` are more values of
|
||||
`country` to accept. A `place:` token the gazetteer does not know is
|
||||
matched on whole words of the role's folded line (`places.match_sql`),
|
||||
so a town nobody has entered yet is still found."""
|
||||
ids: set[int] = set()
|
||||
found = {c for c in codes if c}
|
||||
clauses: list[str] = []
|
||||
params: list[Any] = []
|
||||
for token in tokens:
|
||||
named = gaz.lookup(token)
|
||||
if named is None:
|
||||
sql, values = places.match_sql(token, place=f"{table}.place_fold",
|
||||
country=f"{table}.country")
|
||||
clauses.append(sql)
|
||||
params += values
|
||||
continue
|
||||
under = gaz.under(named)
|
||||
ids |= under
|
||||
found |= gaz.codes(under)
|
||||
if ids:
|
||||
clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places "
|
||||
"WHERE place_id IN (SELECT value FROM json_each(?)))")
|
||||
params.insert(0, json.dumps(sorted(ids)))
|
||||
if found:
|
||||
clauses.insert(1 if ids else 0,
|
||||
f"{table}.country IN (SELECT value FROM json_each(?))")
|
||||
params.insert(1 if ids else 0, json.dumps(sorted(found)))
|
||||
return f"({' OR '.join(clauses) or '0'})", params
|
||||
|
||||
|
||||
def _known_places(tokens: Sequence[str]) -> None:
|
||||
"""ValueError for a token that names nothing, which would otherwise match
|
||||
no role without saying why."""
|
||||
for token in tokens:
|
||||
if not places.known(token):
|
||||
raise ValueError(f"{token!r} is not a place this app knows")
|
||||
sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country")
|
||||
clauses.append(sql)
|
||||
params += values
|
||||
return f"({' OR '.join(clauses)})", params
|
||||
|
||||
|
||||
def _text_rank(description: str, teaser: str) -> int:
|
||||
"""What a role is embedded from, as a rank: 2 its description, 1 its
|
||||
teaser, 0 its title alone."""
|
||||
return 2 if description else 1 if teaser else 0
|
||||
|
||||
|
||||
def _int_or_none(value: Any) -> int | None:
|
||||
@@ -351,6 +399,8 @@ class Store:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
#: What "now" is. Replaced by tests that need two writes a second apart.
|
||||
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
||||
#: The gazetteer as last read, with the `gazetteer.VERSION` it was read at.
|
||||
self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None
|
||||
with self.transaction() as conn:
|
||||
# Persistent: recorded in the file, inherited by every connection.
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
@@ -364,7 +414,7 @@ class Store:
|
||||
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
||||
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
|
||||
registered on every one, since the place filter and the place gate
|
||||
call it (`places.match_sql`)."""
|
||||
call it for a town the gazetteer does not know (`_places_sql`)."""
|
||||
conn = sqlite3.connect(self.path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
places.register(conn)
|
||||
@@ -400,6 +450,103 @@ class Store:
|
||||
def now(self) -> str:
|
||||
return _iso(self.clock())
|
||||
|
||||
# ------------------------------------------------------------- gazetteer
|
||||
|
||||
def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer:
|
||||
"""The gazetteer, read again only when its tables have grown, which
|
||||
another process may have done: one cheap query a call."""
|
||||
version = tuple(conn.execute(gazetteer.VERSION).fetchone())
|
||||
if self._gaz is None or self._gaz[0] != version:
|
||||
self._gaz = (version, gazetteer.load(conn))
|
||||
return self._gaz[1]
|
||||
|
||||
def gazetteer(self) -> places.Gazetteer:
|
||||
with closing(self._connect()) as conn:
|
||||
return self._gazetteer(conn)
|
||||
|
||||
def role_places(self, uid: str) -> list[RolePlace]:
|
||||
"""One role's places in the host's order, a copy's after the role's own."""
|
||||
return [RolePlace.from_row(r) for r in self._rows(
|
||||
"""SELECT rp.position, rp.text, rp.place_id, rp.url,
|
||||
COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind
|
||||
FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id
|
||||
WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""",
|
||||
(uid,))]
|
||||
|
||||
def unresolved_places(self, open_only: bool = True,
|
||||
limit: int | None = None) -> list[tuple[str, str, int]]:
|
||||
"""`(fold, text, roles)` for every place text no place of the
|
||||
gazetteer resolves, most roles first: the list a person works through
|
||||
with `add_place_alias` and `add_place`. `text` is one spelling of the
|
||||
fold. A text that names no place ("Remote", `places.placeless`) is
|
||||
left out; with `open_only`, so are closed roles and copies."""
|
||||
where = "rp.place_id IS NULL"
|
||||
if open_only:
|
||||
where += " AND r.closed_at IS NULL AND r.dup_of IS NULL"
|
||||
rows = self._rows(
|
||||
f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n
|
||||
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
||||
WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""")
|
||||
found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows
|
||||
if not places.placeless(r["fold"])]
|
||||
return found[:limit] if limit is not None else found
|
||||
|
||||
def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int:
|
||||
"""A place by id, or by a token that names exactly one place."""
|
||||
if isinstance(place, int):
|
||||
if place not in gaz.places:
|
||||
raise ValueError(f"there is no place {place}")
|
||||
return place
|
||||
named = gaz.lookup(place) or frozenset()
|
||||
if len(named) != 1:
|
||||
raise ValueError(f"{place!r} names {len(named)} places, not one")
|
||||
return next(iter(named))
|
||||
|
||||
def add_place_alias(self, text: str, place: int | str) -> int:
|
||||
"""Write `text` as a spelling of `place` (an id, or a token such as
|
||||
`place:copenhagen` or `country:DK` naming one place), and resolve again
|
||||
every role with a place text that holds it. Returns how many roles now
|
||||
resolve differently."""
|
||||
folded = places.fold(text)
|
||||
if not folded:
|
||||
raise ValueError("an alias needs a letter or a digit")
|
||||
with self.transaction() as conn:
|
||||
place_id = self._place_id(self._gazetteer(conn), place)
|
||||
conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
||||
(folded, place_id))
|
||||
return gazetteer.relink(conn, self._gazetteer(conn), folded)
|
||||
|
||||
def add_place(self, name: str, kind: str, parent: int | str,
|
||||
aliases: Sequence[str] = ()) -> int:
|
||||
"""Add a city or a region the gazetteer lacks, in `parent` (an id or a
|
||||
token), written as `name` and each of `aliases`, and resolve again the
|
||||
roles with a place text that holds one of them. Returns its id."""
|
||||
if kind not in ("city", "region"):
|
||||
raise ValueError(f"a place added by hand is a city or a region, not {kind!r}")
|
||||
folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f))
|
||||
if not folds:
|
||||
raise ValueError("a place needs a name")
|
||||
with self.transaction() as conn:
|
||||
parent_id = self._place_id(self._gazetteer(conn), parent)
|
||||
if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?",
|
||||
(kind, folds[0])).fetchone():
|
||||
raise ValueError(f"there is already a {kind} {name!r}")
|
||||
place_id = int(conn.execute(
|
||||
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
|
||||
(kind, folds[0], name.strip(), parent_id)).lastrowid or 0)
|
||||
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
||||
[(f, place_id) for f in folds])
|
||||
gaz = self._gazetteer(conn)
|
||||
for folded in folds:
|
||||
gazetteer.relink(conn, gaz, folded)
|
||||
return place_id
|
||||
|
||||
def relink_places(self) -> int:
|
||||
"""Resolve every role's own places again, after the gazetteer was
|
||||
seeded anew. Returns how many roles now resolve differently."""
|
||||
with self.transaction() as conn:
|
||||
return gazetteer.relink(conn, self._gazetteer(conn))
|
||||
|
||||
# ---------------------------------------------------------------- roles
|
||||
|
||||
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
|
||||
@@ -426,19 +573,32 @@ class Store:
|
||||
alone for good, whatever the plugin's raw guess now says — see
|
||||
`set_extracted` for the other half of the rule.
|
||||
|
||||
`seniority` is the title's level (`seniority.from_title`) until
|
||||
extract has read the role and extract's from then on, as `function`
|
||||
is. The classifier is called here rather than in a plugin, so a role
|
||||
from any source gets it, the way `country` comes from `place` here
|
||||
too. A level the board states in `raw` is a board field and wins over
|
||||
both. `place_fold` is written with `place`, under the same keep-filled
|
||||
rule.
|
||||
`seniority` is the first filled of three levels. `seniority_model`
|
||||
is extract's (`set_extracted`) and never written here.
|
||||
`seniority_stated` is the host's word as `raw["seniority"]` carries
|
||||
it, already in `SENIORITIES` (anything else is ''), and
|
||||
`seniority_title` is `seniority.from_title`; both are rewritten every
|
||||
pull. The classifier is called here rather than in a plugin, so a
|
||||
role from any source gets it, the way `country` comes from `place`.
|
||||
|
||||
The role's places (`role_places`) are the texts of `raw["places"]`,
|
||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
|
||||
against the gazetteer; `place` is the line shown, the texts joined
|
||||
when `raw` has none. Like `place`, a pull with no place leaves them
|
||||
as they are. `place_fold` is written with `place`, under the same
|
||||
keep-filled rule. `teaser` is kept-filled too.
|
||||
|
||||
A role is embedded from its description, else its teaser, else its
|
||||
title (`app.jobs.embed.document`). When a pull brings a better one of
|
||||
these than the role had, its vector is dropped so the next `embed`
|
||||
reads it.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
raw = item.raw or {}
|
||||
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
||||
place = str(raw.get("place") or "")
|
||||
place_texts = gazetteer.texts(raw)
|
||||
place = str(raw.get("place") or "") or "; ".join(place_texts)
|
||||
stamp = self.now()
|
||||
published = _iso(item.published_at)
|
||||
row: dict[str, Any] = {
|
||||
@@ -463,12 +623,16 @@ class Store:
|
||||
"created_at": stamp,
|
||||
}
|
||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||
row["place"] = place
|
||||
row["country"] = row["country"] or country_of(place)
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
stated = str(raw.get("seniority") or "")
|
||||
row["seniority"] = stated or from_title(item.title)
|
||||
row["seniority_stated"] = stated if stated in SENIORITIES else ""
|
||||
row["seniority_title"] = from_title(item.title)
|
||||
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
|
||||
row["teaser"] = str(raw.get("teaser") or "")
|
||||
row["place_fold"] = places.fold(row["place"])
|
||||
# A board's own pay line, one per pull like title or published_at —
|
||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||
@@ -476,13 +640,13 @@ class Store:
|
||||
row["salary_text"] = str(raw.get("salary_text") or "")
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", *_RAW_TEXT)
|
||||
"feed_text", "original_url", "teaser", *_RAW_TEXT)
|
||||
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
||||
"salary_text")]
|
||||
"salary_text", "seniority_stated", "seniority_title")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
@@ -493,27 +657,29 @@ class Store:
|
||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||
"ELSE roles.function END")
|
||||
# The title's level follows the title until extract has read the
|
||||
# role, and is extract's from then on; a board's stated level wins.
|
||||
updates.append("seniority=excluded.seniority" if stated else
|
||||
"seniority=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN excluded.seniority ELSE roles.seniority END")
|
||||
# The model's level, else the host's, else the title's.
|
||||
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
|
||||
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
|
||||
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
||||
"THEN excluded.place_fold ELSE roles.place_fold END")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
with self.transaction() as conn:
|
||||
before = conn.execute("SELECT description FROM roles WHERE uid=?",
|
||||
before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?",
|
||||
(row["uid"],)).fetchone()
|
||||
conn.execute(
|
||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||
# A role listed without its text is embedded from its title, and
|
||||
# `embed` encodes a role once. When the text arrives on a later
|
||||
# pull, as it does for a host that states it on the role's own
|
||||
# page, the vector goes, so the next `embed` reads the text.
|
||||
if before is not None and not before["description"] and row["description"]:
|
||||
if place_texts:
|
||||
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
|
||||
# `embed` encodes a role once, from the best text it has. When a
|
||||
# later pull brings a better one, as a host that states the text
|
||||
# on the role's own page does, the vector goes, so the next
|
||||
# `embed` reads it.
|
||||
if before is not None and _text_rank(row["description"] or before["description"],
|
||||
row["teaser"] or before["teaser"]) \
|
||||
> _text_rank(before["description"], before["teaser"]):
|
||||
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
|
||||
return before is None
|
||||
|
||||
@@ -533,6 +699,7 @@ class Store:
|
||||
role = ScoredRole.from_row(row)
|
||||
role.tags = self._tags_of(uid)
|
||||
role.copies = self._copies_of(uid)
|
||||
role.places = self.role_places(uid)
|
||||
return role
|
||||
|
||||
def _copies_of(self, uid: str) -> list[RoleCopy]:
|
||||
@@ -685,7 +852,7 @@ class Store:
|
||||
|
||||
`company_keys` keeps those exact companies. `places` keeps a role any
|
||||
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
|
||||
`place:malmo`), matched on whole folded words; a token that names
|
||||
`place:malmo`), over the gazetteer (`_places_sql`); a token that names
|
||||
nothing is a ValueError. `seniority` keeps the roles at any of those
|
||||
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
|
||||
with no level, which is most of them. An empty list is no filter.
|
||||
@@ -934,8 +1101,8 @@ class Store:
|
||||
else:
|
||||
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
|
||||
if places:
|
||||
# Last, so the cheaper terms have cut the rows it is called on.
|
||||
clause, values = _places_sql(places)
|
||||
_known_places(places)
|
||||
clause, values = _places_sql(places, self.gazetteer())
|
||||
where.append(clause)
|
||||
params += values
|
||||
sql = f"FROM roles {_WITH_VOTES}"
|
||||
@@ -1087,9 +1254,9 @@ class Store:
|
||||
non-empty answer overwrites it outright rather than only filling an
|
||||
empty column — see `upsert_role`'s docstring for the other half of
|
||||
the rule that then keeps a later pull from putting the guess back.
|
||||
`seniority` is the other: `upsert_role` fills it from the title, so a
|
||||
non-empty answer replaces it, unless the board stated a level in
|
||||
`raw`."""
|
||||
`seniority` is the other: a non-empty answer is `seniority_model`,
|
||||
which comes before the stated and the title's level, so it becomes
|
||||
`seniority` too."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
@@ -1103,8 +1270,8 @@ class Store:
|
||||
if function:
|
||||
sets["function"] = function
|
||||
level = str(fields.get("seniority") or "")
|
||||
if level and not json.loads(row["raw"] or "{}").get("seniority"):
|
||||
sets["seniority"] = level
|
||||
if level:
|
||||
sets["seniority_model"] = sets["seniority"] = level
|
||||
for name in _RAW_INT:
|
||||
number = _int_or_none(fields.get(name))
|
||||
if number is not None and row[name] is None:
|
||||
@@ -1112,10 +1279,12 @@ class Store:
|
||||
languages = [str(code) for code in fields.get("languages") or []]
|
||||
if languages and json.loads(row["languages"] or "[]") == []:
|
||||
sets["languages"] = json.dumps(languages)
|
||||
filled = len(sets)
|
||||
filled = len(sets) - ("seniority" in sets)
|
||||
if "place" in sets:
|
||||
sets["place_key"] = place_key(sets["place"])
|
||||
sets["place_fold"] = places.fold(sets["place"])
|
||||
gazetteer.write(conn, self._gazetteer(conn), uid,
|
||||
gazetteer.texts({"place": sets["place"]}))
|
||||
sets["extracted_at"] = self.now()
|
||||
conn.execute(
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
||||
@@ -1375,8 +1544,9 @@ class Store:
|
||||
as a column so a gated role keeps its row and reason."""
|
||||
columns: list[str] = []
|
||||
params: list[Any] = []
|
||||
gaz = self.gazetteer()
|
||||
for n, gate in enumerate(gates):
|
||||
empty, match, values = _gate_sql(gate)
|
||||
empty, match, values = _gate_sql(gate, gaz)
|
||||
columns.append(
|
||||
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
||||
params += values
|
||||
|
||||
@@ -164,13 +164,17 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
||||
store.upsert_role(item)
|
||||
store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))])
|
||||
with sqlite3.connect(path) as conn:
|
||||
for table in ("role_places", "place_includes", "place_aliases", "places"):
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
conn.execute("DROP INDEX idx_roles_dup_of")
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
|
||||
conn.execute("PRAGMA user_version = 17")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [18]
|
||||
assert migrated.migrations_applied == [18, 19]
|
||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
"""The gazetteer: its seed, the resolver, and a role's places in the store.
|
||||
|
||||
The store tests go through `Store`, as the feed and the gates do. The
|
||||
real-line test compares the join over the gazetteer with the whole-word rule
|
||||
it replaced (`places.matches`) on the live lines `test_places` holds: the
|
||||
join must find every role the word rule found, but for the lines listed where
|
||||
the word rule was wrong.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from desk.models import Item
|
||||
from app import places
|
||||
from app.models import Gate, _CITIES, _COUNTRY_NAMES
|
||||
from app.places import Gazetteer, fold, seed
|
||||
from app.storage import Store
|
||||
|
||||
from test_places import LINES, ORACLE_TOKENS
|
||||
|
||||
|
||||
def gazetteer() -> Gazetteer:
|
||||
seeds, includes = seed()
|
||||
ids: dict[tuple[str, str], int] = {}
|
||||
rows, aliases = [], []
|
||||
for n, s in enumerate(seeds, 1):
|
||||
ids[(s.kind, s.key)] = n
|
||||
rows.append((n, s.kind, s.key, s.name, ids[s.parent] if s.parent else None))
|
||||
aliases += [(a, n) for a in s.aliases]
|
||||
return Gazetteer(rows, aliases, [(ids[a], ids[b]) for a, b in includes])
|
||||
|
||||
|
||||
GAZ = gazetteer()
|
||||
|
||||
|
||||
def names(text: str) -> list[str]:
|
||||
return [GAZ.places[p].name for p in GAZ.resolve(text)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- seed
|
||||
|
||||
def test_the_seed_holds_the_dictionaries_in_code() -> None:
|
||||
seeds, includes = seed()
|
||||
kinds = Counter(s.kind for s in seeds)
|
||||
assert kinds["continent"] == 1
|
||||
assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line)
|
||||
keys = {(s.kind, s.key) for s in seeds}
|
||||
assert {("region", key) for key in places.REGIONS} <= keys
|
||||
assert {("region", "oresund"), ("region", "skane")} in [set(i) for i in includes]
|
||||
folds = {a for s in seeds for a in s.aliases}
|
||||
assert set(_CITIES) - {"region hovedstaden"} <= folds
|
||||
assert {"kobenhavn", "kopenhamn", "copenhagen", "skane", "emea", "eu"} <= folds
|
||||
|
||||
|
||||
def test_parents_come_before_their_children() -> None:
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for s in seed()[0]:
|
||||
assert s.parent is None or s.parent in seen, s
|
||||
seen.add((s.kind, s.key))
|
||||
|
||||
|
||||
def test_copenhagen_is_in_the_capital_region_in_denmark_in_europe() -> None:
|
||||
copenhagen = GAZ.by_key[("city", "copenhagen")]
|
||||
above = {GAZ.places[p].key for p in GAZ.ancestors(copenhagen)}
|
||||
assert above == {"hovedstaden", "oresund", "DK", "europe"}
|
||||
assert GAZ.country(copenhagen) == "DK"
|
||||
assert GAZ.by_key[("country", "US")] not in GAZ.under({GAZ.by_key[("continent", "europe")]})
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ resolving
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
("Aarhus N, Region Central Jutland, Denmark", ["Aarhus"]),
|
||||
("København K", ["Copenhagen"]),
|
||||
("Copenhagen Office", ["Copenhagen"]),
|
||||
("Malmö/Lund", ["Malmö", "Lund"]),
|
||||
("Aalborg, Copenhagen, Kolding", ["Aalborg", "Copenhagen", "Kolding"]),
|
||||
("Paderborn, Germany", ["Germany"]),
|
||||
("Skåne län, Sweden", ["Skåne"]),
|
||||
("London, Ontario", ["Ontario"]),
|
||||
("Bergen op Zoom, Netherlands", ["Netherlands"]),
|
||||
("Mexicali, BAJA CALIFORNIA, Mexico", ["Mexico"]),
|
||||
("Paris, TX", ["United States"]),
|
||||
("Remote (DE)", ["Germany"]),
|
||||
("London OR Paris OR Germany", ["London", "Paris", "Germany"]),
|
||||
("Allerød or Stockholm (SE)", ["Allerød", "Stockholm"]),
|
||||
("Finland, København, Litauen, Polen, Sverige", ["Finland", "Copenhagen", "Sweden"]),
|
||||
("Remote, EU or US", ["Europe", "United States"]),
|
||||
("Home based - EMEA", ["Europe"]),
|
||||
("Luxembourg", ["Luxembourg"]),
|
||||
("Nordhavn og mulighed for hjemmearbejde", ["Nordhavn"]),
|
||||
("Remote", []),
|
||||
("Oberkochen", []),
|
||||
("", []),
|
||||
])
|
||||
def test_a_text_resolves_to_the_places_it_names(text: str, expected: list[str]) -> None:
|
||||
assert names(text) == expected
|
||||
|
||||
|
||||
def test_luxembourg_the_town_is_in_luxembourg_the_country() -> None:
|
||||
town = GAZ.by_key[("city", "luxembourg")]
|
||||
assert GAZ.resolve("Luxembourg") == (town,)
|
||||
assert GAZ.country(town) == "LU"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["Remote", "Hybrid", "Multiple Locations", "2 Locations",
|
||||
"Remote job", "", " "])
|
||||
def test_a_text_of_how_a_role_is_worked_is_no_place(text: str) -> None:
|
||||
assert places.placeless(text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Jena"])
|
||||
def test_a_text_with_a_name_is_a_place(text: str) -> None:
|
||||
assert not places.placeless(text)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- tokens
|
||||
|
||||
def test_a_token_names_places_before_what_lies_in_them() -> None:
|
||||
assert GAZ.lookup("country:DK") == {GAZ.by_key[("country", "DK")]}
|
||||
assert GAZ.lookup("country:EU") == {GAZ.by_key[("continent", "europe")]}
|
||||
assert GAZ.lookup("region:Skåne") == {GAZ.by_key[("region", "skane")]}
|
||||
assert GAZ.lookup("place:kobenhavn") == {GAZ.by_key[("city", "copenhagen")]}
|
||||
assert GAZ.lookup("place:denmark") == {GAZ.by_key[("country", "DK")]}
|
||||
assert GAZ.lookup("place:oberkochen") is None
|
||||
assert GAZ.lookup("country:ZZ") is None
|
||||
|
||||
|
||||
def test_europe_is_every_european_country_and_eu() -> None:
|
||||
under = GAZ.under(GAZ.lookup("country:EU") or ())
|
||||
codes = GAZ.codes(under)
|
||||
assert {"DK", "SE", "DE", "GB", "CH", "EU"} <= codes
|
||||
assert not {"US", "CN"} & codes
|
||||
|
||||
|
||||
def test_oresund_holds_both_capital_regions() -> None:
|
||||
under = GAZ.under(GAZ.lookup("region:oresund") or ())
|
||||
assert {GAZ.by_key[("city", "malmo")], GAZ.by_key[("city", "copenhagen")]} <= under
|
||||
assert GAZ.by_key[("city", "aarhus")] not in under
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- store
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Iterator[Store]:
|
||||
yield Store(tmp_path / "roledesk.db")
|
||||
|
||||
|
||||
def posting(source_id: str, *, title: str = "Backend Engineer", body: str = "",
|
||||
**raw: object) -> Item:
|
||||
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
||||
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw=dict(raw))
|
||||
item.source = "ats"
|
||||
return item
|
||||
|
||||
|
||||
def texts(store: Store, uid: str) -> list[tuple[int, str, str]]:
|
||||
return [(p.position, p.text, p.name) for p in store.role_places(uid)]
|
||||
|
||||
|
||||
def test_a_fresh_file_is_seeded(store: Store) -> None:
|
||||
gaz = store.gazetteer()
|
||||
assert len(gaz.places) == len(GAZ.places)
|
||||
assert ("city", "copenhagen") in gaz.by_key
|
||||
|
||||
|
||||
def test_a_joined_line_is_several_places(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
|
||||
uid = posting("1").uid
|
||||
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
|
||||
(2, "Warsaw", "Warsaw")]
|
||||
role = store.role(uid)
|
||||
assert role is not None and role.place == "Copenhagen; London; Warsaw"
|
||||
assert [p.name for p in role.places] == ["Copenhagen", "London", "Warsaw"]
|
||||
|
||||
|
||||
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Berlin, Munich", places=[
|
||||
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
|
||||
{"text": "Remote (DE)", "remote": True}]))
|
||||
assert texts(store, posting("1").uid) == [
|
||||
(0, "Berlin", "Berlin"), (1, "Munich", "Munich"), (2, "Remote (DE)", "Germany")]
|
||||
|
||||
|
||||
def test_places_without_a_line_make_the_line(store: Store) -> None:
|
||||
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
|
||||
{"text": "Malmö", "remote": None}]))
|
||||
role = store.role(posting("1").uid)
|
||||
assert role is not None
|
||||
assert (role.place, role.country) == ("Lund; Malmö", "SE")
|
||||
|
||||
|
||||
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Aarhus"))
|
||||
store.upsert_role(posting("1"))
|
||||
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
|
||||
store.upsert_role(posting("1", place="Odense"))
|
||||
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
|
||||
|
||||
|
||||
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Aarhus"))
|
||||
uid = posting("1").uid
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
||||
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
|
||||
store.upsert_role(posting("1", place="Aalborg"))
|
||||
assert [(p.text, p.url) for p in store.role_places(uid)] == [
|
||||
("Aalborg", ""), ("Odense", "https://copy")]
|
||||
|
||||
|
||||
def test_an_extracted_place_is_resolved(store: Store) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
uid = posting("1").uid
|
||||
store.set_extracted(uid, {"place": "Lund, Sweden"})
|
||||
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
|
||||
|
||||
|
||||
# ---------------------------------------------------------- filter and gates
|
||||
|
||||
def uids(store: Store, *tokens: str) -> set[str]:
|
||||
return {r.uid for r in store.roles(limit=100_000, places=list(tokens), by_score=False)}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def several(store: Store) -> dict[str, str]:
|
||||
lines = {"cph": "Copenhagen; London", "eu": "Berlin, Germany; Paris, France",
|
||||
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
|
||||
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
|
||||
for key, line in lines.items():
|
||||
store.upsert_role(posting(key, place=line))
|
||||
return {key: posting(key).uid for key in lines}
|
||||
|
||||
|
||||
def test_a_country_is_every_place_in_it(store: Store, several: dict[str, str]) -> None:
|
||||
assert uids(store, "country:DK") == {several["cph"], several["billund"]}
|
||||
assert uids(store, "country:SE") == {several["malmo"], several["lund"], several["remote"]}
|
||||
assert uids(store, "country:DE") == {several["eu"]}
|
||||
assert uids(store, "country:GB") == {several["cph"]}
|
||||
|
||||
|
||||
def test_europe_is_every_european_role(store: Store, several: dict[str, str]) -> None:
|
||||
assert uids(store, "country:EU") == set(several.values()) - {several["ny"], several["zeiss"]}
|
||||
|
||||
|
||||
def test_a_region_is_every_town_in_it(store: Store, several: dict[str, str]) -> None:
|
||||
assert uids(store, "region:skane") == {several["malmo"], several["lund"]}
|
||||
assert uids(store, "region:oresund") == {several["malmo"], several["lund"], several["cph"]}
|
||||
assert uids(store, "place:skane") == uids(store, "region:skane")
|
||||
|
||||
|
||||
def test_a_town_is_its_place_and_lund_is_not_in_billund(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
assert uids(store, "place:lund") == {several["lund"]}
|
||||
assert uids(store, "place:kobenhavn") == {several["cph"]}
|
||||
|
||||
|
||||
def test_a_town_the_gazetteer_lacks_is_found_by_its_words(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
assert uids(store, "place:oberkochen") == {several["zeiss"]}
|
||||
|
||||
|
||||
def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
store.roles(places=["region:atlantis"])
|
||||
|
||||
|
||||
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
|
||||
store.upsert_role(posting("1", country="DK"))
|
||||
assert uids(store, "country:DK") == {posting("1").uid}
|
||||
|
||||
|
||||
def gates(store: Store, gate: Gate) -> dict[str, int]:
|
||||
agent = store.save_agent("A", gates=[gate])
|
||||
return {c.uid: c.gates[0] for c in store.score_candidates(agent.id, agent.gates)}
|
||||
|
||||
|
||||
def test_a_country_gate_is_met_by_any_place_in_the_country(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
found = gates(store, Gate(field="country", values=["DK", "SE"], mode="any"))
|
||||
assert {uid for uid, g in found.items() if g == 1} == {
|
||||
several[k] for k in ("cph", "billund", "malmo", "lund", "remote")}
|
||||
# Only a role with neither a country nor a known place has no evidence.
|
||||
assert {uid for uid, g in found.items() if g == -1} == {several["zeiss"]}
|
||||
assert found[several["ny"]] == 0 and found[several["eu"]] == 0
|
||||
|
||||
|
||||
def test_the_eu_value_of_a_country_gate_is_europe(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
found = gates(store, Gate(field="country", values=["EU"]))
|
||||
assert found[several["eu"]] == 1 and found[several["cph"]] == 1
|
||||
assert found[several["ny"]] == 0
|
||||
|
||||
|
||||
def test_a_place_gate_matches_over_the_gazetteer(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
found = gates(store, Gate(field="place", values=["Malmö", "Lund", "oberkochen"],
|
||||
mode="any"))
|
||||
assert {uid for uid, g in found.items() if g == 1} == {
|
||||
several["malmo"], several["lund"], several["zeiss"]}
|
||||
|
||||
|
||||
# ------------------------------------------------------------ unresolved texts
|
||||
|
||||
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
|
||||
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
|
||||
store.upsert_role(posting(str(n), place=line))
|
||||
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
|
||||
("jena", "Jena", 1)]
|
||||
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
|
||||
|
||||
|
||||
def test_an_alias_links_what_it_names(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Kbh. K"))
|
||||
store.upsert_role(posting("2", place="Copenhagen"))
|
||||
uid = posting("1").uid
|
||||
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
|
||||
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
|
||||
assert texts(store, uid) == [(0, "Kbh. K", "Copenhagen")]
|
||||
assert store.unresolved_places() == []
|
||||
assert uids(store, "country:DK") == {uid, posting("2").uid}
|
||||
# The second store, another process, sees the new alias too.
|
||||
assert Store(store.path).gazetteer().lookup("place:kbh") == \
|
||||
store.gazetteer().lookup("place:copenhagen")
|
||||
|
||||
|
||||
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Oberkochen; Jena"))
|
||||
store.upsert_role(posting("2", place="Paderborn, Germany"))
|
||||
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
|
||||
store.add_place("Paderborn", "city", "country:DE")
|
||||
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
|
||||
assert texts(store, posting("2").uid) == [(0, "Paderborn, Germany", "Paderborn")]
|
||||
assert uids(store, "place:oberkochen") == {posting("1").uid}
|
||||
assert store.gazetteer().country(oberkochen) == "DE"
|
||||
with pytest.raises(ValueError):
|
||||
store.add_place("Oberkochen", "city", "country:DE")
|
||||
with pytest.raises(ValueError):
|
||||
store.add_place("Atlantis", "country", "country:EU")
|
||||
|
||||
|
||||
def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
store.add_place_alias("Kbh", "place:atlantis")
|
||||
with pytest.raises(ValueError):
|
||||
store.add_place_alias("!!", "place:copenhagen")
|
||||
|
||||
|
||||
def test_relink_places_resolves_every_role_again(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Oberkochen"))
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
conn.execute("UPDATE role_places SET place_id = "
|
||||
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
|
||||
assert store.relink_places() == 1
|
||||
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "")]
|
||||
|
||||
|
||||
# --------------------------------------------------------- against the words
|
||||
|
||||
#: Lines of `test_places.LINES` the word rule found for a token and the
|
||||
#: gazetteer does not, each rightly: the London in Canada, and a town named
|
||||
#: after Zurich in Illinois.
|
||||
NOT_FOUND_ANY_MORE: dict[str, set[str]] = {
|
||||
"country:GB": {"London, Canada; Remote, United States; Oak Brook, United States"},
|
||||
"place:london": {"London, Canada; Remote, United States; Oak Brook, United States"},
|
||||
"place:zurich": {"Lake Zurich, Illinois, United States"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tuple[str, str]]]:
|
||||
store = Store(tmp_path_factory.mktemp("real") / "roledesk.db")
|
||||
lines = {}
|
||||
for n, (place, country) in enumerate(LINES):
|
||||
item = posting(f"line{n}", place=place, country=country)
|
||||
store.upsert_role(item)
|
||||
lines[item.uid] = (place, country)
|
||||
return store, lines
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token", ORACLE_TOKENS)
|
||||
def test_the_gazetteer_finds_what_the_words_found(
|
||||
real: tuple[Store, dict[str, tuple[str, str]]], token: str) -> None:
|
||||
store, lines = real
|
||||
by_words = {uid for uid, (place, country) in lines.items()
|
||||
if places.matches(fold(place), country, token)}
|
||||
by_places = uids(store, token)
|
||||
lost = {lines[uid][0] for uid in by_words - by_places}
|
||||
assert lost <= NOT_FOUND_ANY_MORE.get(token, set()), (token, lost)
|
||||
+139
-12
@@ -14,8 +14,8 @@ from desk.models import Item
|
||||
import numpy as np
|
||||
|
||||
from app.models import (
|
||||
LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key,
|
||||
posted_at, posted_host, title_key,
|
||||
LEVELS, Application, Gate, Limits, Prompt, company_key, place_key,
|
||||
posted_at, posted_host, title_key, trusted_posted,
|
||||
)
|
||||
from app.storage.store import pack
|
||||
from app.storage import LATEST, Store
|
||||
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 18
|
||||
assert LATEST == 19
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -88,8 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
|
||||
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
|
||||
|
||||
|
||||
def _undo_v19(conn: sqlite3.Connection) -> None:
|
||||
"""A fresh file put back to the v18 shape: no gazetteer, no role places,
|
||||
no teaser, one level."""
|
||||
for table in ("role_places", "place_includes", "place_aliases", "places"):
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
|
||||
|
||||
def _undo_v18(conn: sqlite3.Connection) -> None:
|
||||
"""A fresh file put back to the v17 shape: no copies."""
|
||||
_undo_v19(conn)
|
||||
conn.execute("DROP INDEX idx_roles_dup_of")
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
|
||||
|
||||
@@ -166,7 +176,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18]
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||
@@ -371,7 +381,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18]
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19]
|
||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||
Prompt(name="Backend", text="python backend")]
|
||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||
@@ -395,7 +405,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [16, 17, 18]
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||
@@ -424,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [17, 18]
|
||||
assert migrated.migrations_applied == [17, 18, 19]
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||
"SELECT uid, place_fold, seniority FROM roles")}
|
||||
@@ -517,7 +527,7 @@ def test_posted_host_reads_board_then_provider() -> None:
|
||||
|
||||
|
||||
def test_posted_at_trusts_only_a_known_host() -> None:
|
||||
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS
|
||||
assert trusted_posted("greenhouse") and not trusted_posted("taleo")
|
||||
early = "2020-01-01T00:00:00+00:00"
|
||||
late = "2026-09-23T09:00:00+00:00"
|
||||
# A trusted host's earlier date wins over first_seen...
|
||||
@@ -527,6 +537,29 @@ def test_posted_at_trusts_only_a_known_host() -> None:
|
||||
assert posted_at(early, late, "") == late
|
||||
|
||||
|
||||
def test_a_reader_that_declares_reads_decides_its_own_date_trust(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`fields.posted` of a declared reader wins over the fallback set, either
|
||||
way; a reader without `READS`, or a provider that is none, keeps it."""
|
||||
import app.models as models
|
||||
|
||||
monkeypatch.setattr(models, "_declared_posted",
|
||||
lambda: {"greenhouse": "listed", "gem": "trusted"})
|
||||
assert not trusted_posted("greenhouse")
|
||||
assert trusted_posted("gem")
|
||||
assert trusted_posted("jobindex")
|
||||
assert not trusted_posted("taleo")
|
||||
|
||||
|
||||
def test_the_declarations_are_read_off_the_readers() -> None:
|
||||
import app.models as models
|
||||
from plugins.readers import READERS
|
||||
|
||||
declared = models._declared_posted()
|
||||
assert set(declared) <= set(READERS)
|
||||
assert all(value in ("trusted", "listed", "none") for value in declared.values())
|
||||
|
||||
|
||||
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
|
||||
first_seen = "2026-09-23T09:00:00+00:00"
|
||||
future = "2026-09-24T00:00:00+00:00" # after first_seen
|
||||
@@ -574,6 +607,92 @@ def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store)
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
|
||||
|
||||
def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
|
||||
store: Store) -> None:
|
||||
from app.jobs.embed import document
|
||||
|
||||
listed = posting(teaser="Build the payments core.")
|
||||
listed.body = ""
|
||||
bare = posting()
|
||||
bare.body = ""
|
||||
store.upsert_role(bare) # the title alone
|
||||
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(listed) # a teaser beats the title
|
||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||
role = store.role(listed.uid)
|
||||
assert role is not None and role.teaser == "Build the payments core."
|
||||
assert document(role) == "Senior Backend Engineer (m/w/d)\nBuild the payments core."
|
||||
assert store.roles_page()[0].excerpt == "Build the payments core."
|
||||
|
||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(bare) # a pull without it keeps it
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
|
||||
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
|
||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||
role = store.role(listed.uid)
|
||||
assert role is not None and document(role).endswith("We build things.")
|
||||
|
||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
|
||||
|
||||
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
|
||||
store.upsert_role(posting("1")) # "Senior Backend Engineer"
|
||||
uid = posting("1").uid
|
||||
|
||||
def levels() -> tuple[str, str, str, str]:
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
return (role.seniority, role.seniority_model, role.seniority_stated,
|
||||
role.seniority_title)
|
||||
|
||||
assert levels() == ("senior", "", "", "senior")
|
||||
store.upsert_role(posting("1", seniority="lead"))
|
||||
assert levels() == ("lead", "", "lead", "senior")
|
||||
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
|
||||
assert levels() == ("senior", "", "", "senior")
|
||||
store.set_extracted(uid, {"seniority": "staff"})
|
||||
store.upsert_role(posting("1", seniority="lead"))
|
||||
assert levels() == ("staff", "staff", "lead", "senior")
|
||||
|
||||
|
||||
def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> None:
|
||||
path = tmp_path / "v18.db"
|
||||
store = Store(path)
|
||||
joined = posting("joined", place="Copenhagen; Malmö")
|
||||
detailed = posting("detailed", place="Aarhus")
|
||||
stated = posting("stated", seniority="lead")
|
||||
extracted = posting("extracted")
|
||||
for item in (joined, detailed, stated, extracted):
|
||||
store.upsert_role(item)
|
||||
with sqlite3.connect(path) as conn:
|
||||
_undo_v19(conn)
|
||||
# A role whose own page was read is pulled with an empty place in raw.
|
||||
conn.execute("UPDATE roles SET raw = '{}' WHERE uid = ?", (detailed.uid,))
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-23', seniority = 'staff' "
|
||||
"WHERE uid = ?", (extracted.uid,))
|
||||
conn.execute("PRAGMA user_version = 18")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [19]
|
||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||
assert migrated.role_places(stated.uid) == []
|
||||
with sqlite3.connect(path) as conn:
|
||||
levels = {uid: rest for uid, *rest in conn.execute(
|
||||
"SELECT uid, seniority, seniority_model, seniority_stated, seniority_title, teaser "
|
||||
"FROM roles")}
|
||||
assert levels == {
|
||||
joined.uid: ["senior", "", "", "senior", ""],
|
||||
detailed.uid: ["senior", "", "", "senior", ""],
|
||||
stated.uid: ["lead", "", "lead", "senior", ""],
|
||||
extracted.uid: ["staff", "staff", "", "senior", ""],
|
||||
}
|
||||
|
||||
|
||||
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
||||
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
|
||||
salary_max=95000, currency="EUR", department="Eng"))
|
||||
@@ -738,12 +857,20 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
|
||||
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None:
|
||||
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
|
||||
store: Store) -> None:
|
||||
store.upsert_role(posting("1", seniority="junior"))
|
||||
uid = posting("1").uid
|
||||
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
|
||||
store.set_extracted(uid, {"seniority": "senior"})
|
||||
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
assert (role.seniority, role.seniority_stated, role.seniority_title,
|
||||
role.seniority_model) == ("junior", "junior", "senior", "")
|
||||
store.set_extracted(uid, {"seniority": "staff"})
|
||||
store.upsert_role(posting("1", seniority="junior"))
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
|
||||
("staff", "junior", "staff")
|
||||
|
||||
|
||||
def test_company_falls_back_to_the_subscription(store: Store) -> None:
|
||||
|
||||
Reference in new issue
Block a user