diff --git a/app/jobs/embed.py b/app/jobs/embed.py index 1ba5f16..6f1f788 100644 --- a/app/jobs/embed.py +++ b/app/jobs/embed.py @@ -2,7 +2,10 @@ The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors` (see `Store.put_vectors`). What gets encoded is the title and the opening of -the description, which is what says what a role is. +the description, which is what says what a role is; while a role has no +description, the teaser its host's list carried, and while it has neither, +the title alone. `Store.upsert_role` drops the vector when a better text +arrives, so the role is encoded again. sentence-transformers is optional, as in newsdesk. Without it this task puts one notice on screen and does nothing; `score` then leaves `fit` out and ranks @@ -55,7 +58,8 @@ def wants_e5_prefix(model: str) -> bool: def document(role: Role) -> str: - return f"{role.title}\n{role.description[:BODY_CHARS]}" + """The title and the description, else the teaser.""" + return f"{role.title}\n{(role.description or role.teaser)[:BODY_CHARS]}" def role_text(model: str, role: Role) -> str: diff --git a/app/models.py b/app/models.py index 2feec97..89752b1 100644 --- a/app/models.py +++ b/app/models.py @@ -12,7 +12,10 @@ from __future__ import annotations import json import re +import sys import unicodedata +from functools import lru_cache +from pathlib import Path from typing import Annotated, Any, Literal, get_args from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator @@ -358,8 +361,9 @@ _REGIONS = ("europe", "emea", "eu", "european union", "eea") #: state or a Canadian province in the same part of the line beats one of #: these ("London, Ontario" is Canada); a country name does not lose to it. _CITIES: dict[str, str] = { - **dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"), - **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"), + **dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck"), "AT"), + **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege", + "luik"), "BE"), **dict.fromkeys(("sofia",), "BG"), **dict.fromkeys(("zagreb",), "HR"), **dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"), @@ -369,7 +373,7 @@ _CITIES: dict[str, str] = { "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", "brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"), **dict.fromkeys(("tallinn", "tartu"), "EE"), - **dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"), + **dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"), **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", "lille", "rennes", "amiens"), "FR"), **dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln", @@ -378,27 +382,30 @@ _CITIES: dict[str, str] = { "mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen", "bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg", "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", - "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"), + "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck"), "DE"), **dict.fromkeys(("athens", "thessaloniki"), "GR"), **dict.fromkeys(("budapest",), "HU"), **dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"), **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", - "florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"), + "florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples", + "napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"), **dict.fromkeys(("riga",), "LV"), **dict.fromkeys(("vilnius", "kaunas"), "LT"), + **dict.fromkeys(("luxembourg", "luxemburg"), "LU"), **dict.fromkeys(("valletta",), "MT"), **dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", "delft"), "NL"), **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"), "PL"), **dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"), - **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"), + **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"), **dict.fromkeys(("belgrade", "beograd"), "RS"), **dict.fromkeys(("bratislava",), "SK"), **dict.fromkeys(("ljubljana",), "SI"), **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", "granada", "santiago de compostela"), "ES"), - **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"), + **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras", + "linkoping", "orebro", "norrkoping", "jonkoping"), "SE"), **dict.fromkeys(("reykjavik",), "IS"), **dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"), **dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk", @@ -408,7 +415,7 @@ _CITIES: dict[str, str] = { "slough", "southampton", "swindon", "liverpool", "redhill", "milton keynes"), "GB"), **dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug", - "sankt gallen", "st gallen"), "CH"), + "sankt gallen", "st gallen", "lucerne", "luzern"), "CH"), **dict.fromkeys(("kyiv", "kiev"), "UA"), **dict.fromkeys(("istanbul", "ankara"), "TR"), **dict.fromkeys(("tirana",), "AL"), @@ -659,9 +666,10 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason) # -------------------------------------------------------------- posting dates -#: Hosts whose date is a creation or first-publication timestamp, safe to -#: read as when a role was posted, checked against live boards on -#: 2026-09-29: +#: Hosts whose date is a creation or first-publication timestamp, for a +#: reader that does not declare `READS` yet (see `trusted_posted`); a reader +#: that declares it says so with `fields.posted="trusted"`. Checked against +#: live boards on 2026-09-29: #: #: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers #: `first_published` over `updated_at`. A live board (adyen) showed a job @@ -708,14 +716,39 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason) #: HR Manager) state a date nobody checked, and Pinpoint states none. #: #: A host not listed here, or a role whose stored `published_at` fails the -#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the -#: safe default this migration is replacing everywhere else. -TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({ +#: sanity check in `posted_at`, keeps `first_seen` as its posted date. +_UNDECLARED_TRUSTED: frozenset[str] = frozenset({ "greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor", "jobindex", "adzuna", "platsbanken", "workday", "successfactors", "oracle", "phenom", "bamboohr", "jobylon", "rippling", }) +@lru_cache(maxsize=1) +def _declared_posted() -> dict[str, str]: + """`fields.posted` of every reader that declares `READS`, by name. Read + once: the readers are loaded when `plugins.readers` is imported. `plugins` + is a top-level folder beside `app`, reached as `app/jobs/extract.py` + reaches it.""" + root = str(Path(__file__).resolve().parents[1]) + if root not in sys.path: + sys.path.insert(0, root) + from plugins.readers import READERS + + return {name: module.READS.fields.posted for name, module in READERS.items() + if getattr(module, "READS", None) is not None} + + +def trusted_posted(host: str) -> bool: + """Whether `host` states a date that is safe to read as when a role was + posted: its reader declares `fields.posted="trusted"`, or, for a reader + without `READS` (or a search provider not yet a reader), it is in + `_UNDECLARED_TRUSTED`.""" + declared = _declared_posted() + if host in declared: + return declared[host] == "trusted" + return host in _UNDECLARED_TRUSTED + + #: A `published_at` before this is a parsing slip or a placeholder, not a #: real posting date: no board here existed before 2015. _POSTED_FLOOR = "2015-01-01T00:00:00+00:00" @@ -732,7 +765,7 @@ def posted_host(raw: dict[str, Any]) -> str: def posted_at(published_at: str, first_seen: str, host: str) -> str: """The date a role counts as posted: the earlier of `first_seen` (when the app itself first stored it) and `published_at` (what the board - claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and + claims), but only when `trusted_posted(host)` and `published_at` is neither absurdly old nor later than `first_seen` — a role cannot be posted after the app already saw it, trusted host or not, which also catches a `published_at` the board reports in the future @@ -740,7 +773,7 @@ def posted_at(published_at: str, first_seen: str, host: str) -> str: `Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings is exact and needs no parsing. A role with no trusted date, or with one that fails the check, keeps `first_seen`.""" - if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen: + if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen: return published_at return first_seen @@ -775,6 +808,20 @@ class RoleCopy(Model): posted_at: str = "" +class RolePlace(Model): + """One place of a role (`role_places`): the host's `text`, and the place + of the gazetteer it resolves to, `place_id` None (and `name` and `kind` + empty) while it resolves to none. `url` is the link of a copy folded into + the role, '' for the role's own places.""" + + position: int = 0 + text: str = "" + place_id: int | None = None + name: str = "" + kind: str = "" + url: str = "" + + class Role(Model): """A row of `roles`: one posting on one board.""" @@ -797,13 +844,20 @@ class Role(Model): #: could be normalised into `salary_min`/`salary_max`/`currency` #: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md). salary_text: str = "" + #: The level shown and filtered on: the first filled of `seniority_model`, + #: `seniority_stated` and `seniority_title`. seniority: str = "" + seniority_model: str = "" + seniority_stated: str = "" + seniority_title: str = "" function: str = "" permit: Permit = "" #: ISO 639-1 codes the posting says are required, lowercase. languages: JsonStrings = [] description: str = "" feed_text: str = "" + #: The short text the host's list carries, until `description` arrives. + teaser: str = "" original_url: str = "" published_at: str #: The date this role counts as posted: see `posted_at`. Used for @@ -838,6 +892,9 @@ class Role(Model): #: The open copies of this role, oldest first, when it is the kept one; #: empty otherwise. Not a column: filled by `Store.role`. copies: list[RoleCopy] = [] + #: Every place of the role, in the host's order. Not a column: filled by + #: `Store.role` from `role_places`. + places: list[RolePlace] = [] @computed_field # type: ignore[prop-decorator] @property diff --git a/app/places.py b/app/places.py index 005619c..0a3e37d 100644 --- a/app/places.py +++ b/app/places.py @@ -19,8 +19,13 @@ several hundred milliseconds a query, as a set intersection on the words of a line it costs a few. `register` gives a connection that answer as the SQL function `place_match`, and `match_sql` returns the clause that calls it. `suggest` turns typed text into tokens for the autocomplete; `label` reads a -chip's text back from the address. This module opens no database; the caller -hands over the places it holds, or its connection. +chip's text back from the address. + +The filter and the gates no longer match words: a role's places are resolved +against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`), +and a token is the places under the place it names. `matches` stays the rule +for a `place:` token the gazetteer does not know. This module opens no +database; the caller hands over the places it holds, or its connection. """ from __future__ import annotations @@ -32,10 +37,12 @@ from collections.abc import Iterable from dataclasses import dataclass from functools import lru_cache -from app.models import _CITIES, _COUNTRY_NAMES, _REGIONS, _place_fold +from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold, + country_of) -__all__ = ["REGIONS", "SQL_FUNCTION", "Region", "Suggestion", "Token", "fold", "known", - "label", "match_sql", "matches", "parse", "register", "suggest", "towns"] +__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed", + "Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse", + "placeless", "register", "seed", "suggest", "towns"] def fold(text: str) -> str: @@ -538,6 +545,9 @@ _PART = re.compile(r"\s*[;|]\s*|\s+(?:or|eller|og)\s+", re.IGNORECASE) #: Danish boards append "and the possibility of working from home". _AND_HOME = re.compile(r"\s+(?:og|samt) mulighed for hjemmearbejde\s*$", re.IGNORECASE) +#: A slash between two places, "Malmö/Lund", "Berlin / Munich". +_SLASH = re.compile(r"\s*/\s*") + #: A postal district after the town, "København K", "Aarhus C", "Odense SV". _DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$") @@ -665,3 +675,301 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \ regions.sort(key=lambda s: (-s.count, s.label)) towns.sort(key=lambda s: (-s.count, s.label)) return (countries + regions + towns)[:limit] + + +# ------------------------------------------------------------------ gazetteer + +#: The kinds of place, widest first: the index is how specific a kind is. +KINDS: tuple[str, ...] = ("continent", "country", "region", "city") + +#: Entries of `_CITIES` that name a region rather than a town. +_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia", + "bay area")) + +#: Words a place text holds when it names how or where-ever a role is worked +#: rather than a place: "Remote", "Multiple Locations", "Home based". A text +#: of these alone is no place, so it is never listed as unresolved. +_PLACELESS = frozenset(( + "remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based", + "from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in", + "multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any", + "first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position")) + + +def placeless(text: str) -> bool: + """Whether a place text names no place at all: empty, or only numbers and + words such as `Remote`, `Hybrid`, `2 Locations`.""" + return all(word in _PLACELESS or word.isdigit() for word in fold(text).split()) + + +@dataclass(frozen=True) +class Seed: + """One place as the code knows it: `key` is unique within `kind` (an ISO + code for a country, a region's key, a town's group), `parent` the + `(kind, key)` of the place it lies in, `aliases` the folds it is written + as.""" + + kind: str + key: str + name: str + parent: tuple[str, str] | None + aliases: tuple[str, ...] + + +def _subdivisions() -> list[tuple[str, str, str]]: + """`(country, code, name)` of every US state and Canadian province.""" + found = [] + for line in _SUBDIVISIONS.split("\n"): + if line: + country, rest = line.split(" ", 1) + for entry in rest.split("|"): + name, code = entry.strip().rsplit(" ", 1) + found.append((country, code, name)) + return found + + +def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]: + """The gazetteer the dictionaries in code describe, parents before their + children, and the inclusions beyond the tree (Øresund holds the Capital + Region and Skåne, which lie in two countries). + + Europe is a continent over the countries of `_EUROPE`, and is also what + `EU`, `EMEA` and `European Union` name. Every country of `_COUNTRY_NAMES` + by its names; the US states and Canadian provinces as regions; the + regions of `REGIONS`; the towns of the alias groups, of the regions and + of `_CITIES`, one place per group (`Copenhagen`, `København`, + `Kopenhamn` are one), in their region when a region lists them and else + in their country.""" + europe = ("continent", "europe") + seeds: dict[tuple[str, str], Seed] = {} + + def add(kind: str, key: str, name: str, parent: tuple[str, str] | None, + aliases: Iterable[str]) -> None: + folds = tuple(dict.fromkeys(a for a in aliases if a)) + if (kind, key) in seeds: + old = seeds[(kind, key)] + folds = tuple(dict.fromkeys((*old.aliases, *folds))) + seeds[(kind, key)] = Seed(kind, key, old.name, old.parent, folds) + else: + seeds[(kind, key)] = Seed(kind, key, name, parent, folds) + + add(*europe, "Europe", None, (fold(name) for name in ("Europe", *_REGIONS))) + for code, names in _COUNTRY_LABELS.items(): + add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code]) + for country, code, name in _subdivisions(): + add("region", f"{country}-{code}", name, ("country", country), (fold(name),)) + for region in REGIONS.values(): + parent = ("country", region.country) if region.country else europe + add("region", region.key, region.label, parent, + (v for name in (region.key, region.label, *region.names) + for v in _variants(fold(name)))) + for name in sorted(_CITY_REGIONS): + add("region", name.replace(" ", "-"), name.title(), ("country", _CITIES[name]), (name,)) + + # Towns, one per group, in the region that lists them first. + town_region: dict[str, str] = {} + for region in REGIONS.values(): + for name in region.towns: + town_region.setdefault(_group(fold(name)), region.key) + folds: dict[str, dict[str, None]] = {} + shown: dict[str, str] = {} + + def add_town(folded: str, name: str) -> None: + key = _group(folded) + folds.setdefault(key, {}).update(dict.fromkeys(_expand(folded))) + shown.setdefault(key, name) + + for spellings in _ALIAS_GROUPS: + for spelling in spellings: + add_town(fold(spelling), spellings[0]) + for region in REGIONS.values(): + for name in region.towns: + add_town(fold(name), name) + for city in _CITIES: + if city not in _CITY_REGIONS and city not in _REGION_BY_NAME: + add_town(city, _TOWN_LABELS.get(city) or city.title()) + for key, forms in folds.items(): + country = next((_CITIES[f] for f in forms if f in _CITIES), "") + if key in town_region: + parent = ("region", town_region[key]) + elif country: + parent = ("country", country) + else: + raise ValueError(f"the town {shown[key]!r} has no country: add it to _CITIES") + add("city", key, shown[key], parent, forms) + + includes = [(("region", region.key), ("region", part)) + for region in REGIONS.values() for part in region.includes] + return list(seeds.values()), includes + + +@dataclass(frozen=True) +class Place: + """A row of `places`.""" + + id: int + kind: str + key: str + name: str + parent_id: int | None + + +class Gazetteer: + """The places, their aliases and what lies in what, as the database holds + them, read into memory: what resolves a role's place text to places and a + filter token to the places under it. Built by the store from its tables, + and rebuilt when they change.""" + + def __init__(self, rows: Iterable[tuple[int, str, str, str, int | None]], + aliases: Iterable[tuple[str, int]], + includes: Iterable[tuple[int, int]] = ()) -> None: + self.places: dict[int, Place] = {r[0]: Place(*r) for r in rows} + self.by_key: dict[tuple[str, str], int] = { + (p.kind, p.key): p.id for p in self.places.values()} + by_fold: dict[str, list[int]] = {} + for folded, place_id in aliases: + by_fold.setdefault(folded, []).append(place_id) + self.aliases: dict[str, tuple[int, ...]] = {f: tuple(ids) for f, ids in by_fold.items()} + self._longest = max((len(f.split()) for f in self.aliases), default=1) + self._up: dict[int, set[int]] = {} + self._down: dict[int, set[int]] = {} + for p in self.places.values(): + if p.parent_id is not None: + self._link(p.parent_id, p.id) + for whole, part in includes: + self._link(whole, part) + self._ancestors: dict[int, frozenset[int]] = {} + self._resolved: dict[str, tuple[int, ...]] = {} + + def _link(self, whole: int, part: int) -> None: + self._up.setdefault(part, set()).add(whole) + self._down.setdefault(whole, set()).add(part) + + def ancestors(self, place_id: int) -> frozenset[int]: + """Every place `place_id` lies in, through parents and inclusions.""" + if (found := self._ancestors.get(place_id)) is None: + seen: set[int] = set() + todo = list(self._up.get(place_id, ())) + while todo: + up = todo.pop() + if up not in seen: + seen.add(up) + todo += self._up.get(up, ()) + found = self._ancestors[place_id] = frozenset(seen) + return found + + def under(self, ids: Iterable[int]) -> frozenset[int]: + """The places and every place that lies in one of them.""" + seen: set[int] = set() + todo = list(ids) + while todo: + place_id = todo.pop() + if place_id not in seen: + seen.add(place_id) + todo += self._down.get(place_id, ()) + return frozenset(seen) + + def country(self, place_id: int) -> str: + """The ISO code of the country a place is in, '' above or outside one.""" + place: Place | None = self.places.get(place_id) + while place is not None and place.kind != "country": + place = self.places.get(place.parent_id) if place.parent_id is not None else None + return place.key if place is not None else "" + + def codes(self, ids: Iterable[int]) -> frozenset[str]: + """The ISO codes of the countries among `ids`, and `EU` with Europe: + what a `roles.country` column holds for a role of one of them.""" + found = set() + for place_id in ids: + place = self.places[place_id] + if place.kind == "country": + found.add(place.key) + elif (place.kind, place.key) == ("continent", "europe"): + found.add("EU") + return frozenset(found) + + def _specific(self, place_id: int) -> int: + return KINDS.index(self.places[place_id].kind) + + def _hits(self, folded: str) -> list[tuple[int, ...]]: + words = folded.split() + hits = [] + i = 0 + while i < len(words): + for n in range(min(self._longest, len(words) - i), 0, -1): + if (ids := self.aliases.get(" ".join(words[i:i + n]))) is not None: + hits.append(ids) + i += n + break + else: + i += 1 + return hits + + def resolve(self, text: str) -> tuple[int, ...]: + """The places one place text names, most specific first from the left: + `Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund` + is Malmö and Lund, `Paderborn, Germany` is Germany, `Remote` is none. + + Every alias the text holds as whole words is a candidate, longest + first. A fold that names several places is the one in the country + `country_of` reads off the text, if one is, and else the most specific + (`Luxembourg` the town before the country). A candidate that another + one lies in is dropped, and so is one outside the one country the text + names, or, naming none, outside the regions it names in one country: + `London, Ontario` is Ontario, `Bergen op Zoom, Netherlands` the + Netherlands, `Mexicali, Baja California, Mexico` Mexico. Several + countries are a list: `London, Spain, Estonia` is all three. When nothing left lies in the country + `country_of` reads (from a state code, `Paris, TX`, or a code in + parentheses, `Remote (DE)`), the text is that country.""" + folded = fold(text) + if (found := self._resolved.get(folded)) is None: + parts = _SLASH.split(_AND_HOME.sub("", text or "")) + found = tuple(dict.fromkeys( + p for part in parts for sub in _PART.split(part) for p in self._resolve(sub))) + self._resolved[folded] = found + return found + + def _resolve(self, text: str) -> list[int]: + """`resolve` for one part of a text, cut where `or`, `/` or `;` end one + place.""" + code = country_of(text) + country = self.by_key.get(("country", code)) if code and code != "EU" else None + within = self.under((country,)) if country is not None else frozenset() + picked: list[int] = [] + for hit in self._hits(fold(text)): + choice = [p for p in hit if p in within] or list(hit) + best = max(choice, key=self._specific) + if best not in picked: + picked.append(best) + # The widest units named hold the rest: one country, else the regions + # of one country. Several countries are a list, not a path. + for kind in ("country", "region"): + named = [p for p in picked if self.places[p].kind == kind] + if len({self.country(p) for p in named}) == 1: + holds = self.under(named) + picked = [p for p in picked if p in holds or self.places[p].kind == "continent" + or any(p in self.ancestors(q) for q in named)] + if named: + break + wider = {a for p in picked for a in self.ancestors(p)} + picked = [p for p in picked if p not in wider] + if country is not None and not any(p in within or p in self.ancestors(country) + for p in picked): + picked = [country] + return picked + + def lookup(self, token: str) -> frozenset[int] | None: + """The places a filter token names, before what lies in them: `country:DK` + is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo` + every place written so (a town, or a region or a country by name). + None when the gazetteer knows no such place.""" + tok = parse(token) + if tok.kind == "country": + key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value) + found = {self.by_key[key]} if key in self.by_key else set() + elif tok.kind == "region": + found = {self.by_key[("region", tok.value)]} if ("region", tok.value) in self.by_key \ + else set(self.aliases.get(tok.value, ())) + else: + found = {p for form in _expand(tok.value) for p in self.aliases.get(form, ())} + return frozenset(found) or None diff --git a/app/storage/__init__.py b/app/storage/__init__.py index b4eea12..d8db82f 100644 --- a/app/storage/__init__.py +++ b/app/storage/__init__.py @@ -3,6 +3,8 @@ schema.py the tables, and the numbered migrations that bring an older file to the current shape store.py `Store`, every SQL statement in the application + gazetteer.py the places tables: seeded from the code, read back, and a + role's places written and relinked Written for roledesk rather than imported from newsdesk: `desk` has no storage package on purpose, and the connection plumbing here is a deliberate copy. diff --git a/app/storage/gazetteer.py b/app/storage/gazetteer.py new file mode 100644 index 0000000..ae3386f --- /dev/null +++ b/app/storage/gazetteer.py @@ -0,0 +1,121 @@ +"""The gazetteer's tables: seeding them from the code, reading them back, and +writing a role's places. + +`places`, `place_aliases` and `place_includes` hold the places +(`app.places.seed` on a new or migrated file, and what a person adds later). +`role_places` holds one row per place of a role: the host's own text, its +fold, and the place it resolves to, NULL for a text the gazetteer does not +know. A text that names several places ("Malmö/Lund") is one row per place, +each with the same `position` and `text`. A row with a `url` is the place of +a copy folded into the role (`app/dupes.py`) and is not the role's own: only +own rows (`url = ''`) are rewritten here. +""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +from app import places + +#: What changes when the gazetteer does: rows are only ever added. +VERSION = ("SELECT (SELECT MAX(rowid) FROM places), (SELECT MAX(rowid) FROM place_aliases), " + "(SELECT COUNT(*) FROM place_includes)") + + +def seed(conn: sqlite3.Connection) -> int: + """Insert what `places.seed` describes, leaving rows already there alone. + Returns how many places were added.""" + seeds, includes = places.seed() + ids = {(kind, key): place_id for place_id, kind, key in + conn.execute("SELECT id, kind, key FROM places")} + added = 0 + for s in seeds: + if (s.kind, s.key) not in ids: + parent = ids[s.parent] if s.parent else None + ids[(s.kind, s.key)] = conn.execute( + "INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)", + (s.kind, s.key, s.name, parent)).lastrowid + added += 1 + conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)", + [(a, ids[(s.kind, s.key)]) for a in s.aliases]) + conn.executemany("INSERT OR IGNORE INTO place_includes (place_id, part_id) VALUES (?, ?)", + [(ids[whole], ids[part]) for whole, part in includes]) + return added + + +def load(conn: sqlite3.Connection) -> places.Gazetteer: + return places.Gazetteer( + [tuple(r) for r in conn.execute("SELECT id, kind, key, name, parent_id FROM places")], + [tuple(r) for r in conn.execute("SELECT fold, place_id FROM place_aliases")], + [tuple(r) for r in conn.execute("SELECT place_id, part_id FROM place_includes")]) + + +def texts(raw: Mapping[str, Any]) -> list[str]: + """A role's place texts from `Item.raw`: the `text` of each of `places` + (`[{"text": ..., "remote": ...}]`, the reader contract), else `place` split + at `;`, which is how the older readers join several places into one line. + Blank and repeated texts are dropped.""" + given = raw.get("places") + if isinstance(given, list) and given: + found = [str((p.get("text") if isinstance(p, Mapping) else p) or "") for p in given] + else: + found = str(raw.get("place") or "").split(";") + return list(dict.fromkeys(t.strip() for t in found if t and t.strip())) + + +def rows(gaz: places.Gazetteer, uid: str, place_texts: Sequence[str] + ) -> list[tuple[str, int, str, str, int | None]]: + """`(uid, position, text, fold, place_id)` for each place of the texts.""" + found: list[tuple[str, int, str, str, int | None]] = [] + for position, text in enumerate(place_texts): + folded = places.fold(text) + found += [(uid, position, text, folded, place_id) + for place_id in (gaz.resolve(text) or (None,))] + return found + + +_INSERT = "INSERT INTO role_places (uid, position, text, fold, place_id) VALUES (?, ?, ?, ?, ?)" + + +def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str, + place_texts: Sequence[str]) -> None: + """Replace one role's own places with these texts, resolved.""" + conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,)) + conn.executemany(_INSERT, rows(gaz, uid, place_texts)) + + +def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer, + by_uid: Iterable[tuple[str, Sequence[str]]]) -> int: + """`write` for many roles in one statement each way. Returns rows written.""" + batch = list(by_uid) + conn.executemany("DELETE FROM role_places WHERE uid = ? AND url = ''", + [(uid,) for uid, _ in batch]) + new = [row for uid, found in batch for row in rows(gaz, uid, found)] + conn.executemany(_INSERT, new) + return len(new) + + +def relink(conn: sqlite3.Connection, gaz: places.Gazetteer, words: str | None = None) -> int: + """Resolve the own places of every role again, or only of the roles with a + place text that holds `words` (a fold) as whole words, after an alias or a + place was added. Returns how many roles now resolve differently.""" + sql = "SELECT uid, position, text, place_id FROM role_places WHERE url = ''" + params: list[Any] = [] + if words: + sql += " AND ' ' || fold || ' ' LIKE ? ESCAPE '\\'" + escaped = words.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + params.append(f"% {escaped} %") + sql = ("SELECT uid, position, text, place_id FROM role_places WHERE url = '' AND uid IN " + f"(SELECT uid FROM ({sql}))") + before: dict[str, dict[tuple[int, str], set[int | None]]] = {} + for uid, position, text, place_id in conn.execute(sql + " ORDER BY uid, position", params): + before.setdefault(uid, {}).setdefault((position, text), set()).add(place_id) + changed = [] + for uid, held in before.items(): + now = {key: set(gaz.resolve(key[1]) or (None,)) for key in held} + if now != held: + changed.append((uid, [text for _, text in sorted(held)])) + write_many(conn, gaz, changed) + return len(changed) diff --git a/app/storage/schema.py b/app/storage/schema.py index 0dd9dee..a57bc01 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -18,8 +18,9 @@ import sqlite3 from collections.abc import Callable from app import places -from app.models import country_of, posted_at, posted_host +from app.models import SENIORITIES, country_of, posted_at, posted_host from app.seniority import from_title +from app.storage import gazetteer log = logging.getLogger(__name__) @@ -44,8 +45,9 @@ CREATE TABLE IF NOT EXISTS roles ( -- require the country: "UK | Remote" is not remote from Berlin. place TEXT NOT NULL DEFAULT '', place_key TEXT NOT NULL DEFAULT '', - -- places.fold(place): what the Place filter and the place gate match - -- whole words against, through the place_match function (app/places.py). + -- places.fold(place): what a place: token the gazetteer does not know is + -- matched against as whole words, through place_match (app/places.py). + -- The filter and the gates otherwise read role_places. place_fold TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', remote TEXT NOT NULL DEFAULT '' @@ -53,7 +55,8 @@ CREATE TABLE IF NOT EXISTS roles ( salary_min INTEGER, salary_max INTEGER, currency TEXT NOT NULL DEFAULT '', - -- seniority.from_title(title) at ingest, until extract reads the posting. + -- The level shown and filtered on: the first filled of seniority_model, + -- seniority_stated and seniority_title (the last columns), written with them. seniority TEXT NOT NULL DEFAULT '', function TEXT NOT NULL DEFAULT '', -- Whether the posting says it sponsors a work permit, and the languages @@ -69,7 +72,7 @@ CREATE TABLE IF NOT EXISTS roles ( -- stored the role; posted_at (app.models.posted_at) is the earlier of -- first_seen and published_at, trusted only for a host known to report a -- creation date rather than one that moves on an edit - -- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order + -- (app.models.trusted_posted) — recency and the feed's newest order -- read posted_at, first_seen stays "when the app found it". closed_at is -- set when a board fetched in full no longer lists the role, and cleared -- if it returns. @@ -100,7 +103,62 @@ CREATE TABLE IF NOT EXISTS roles ( -- precedes, which the migration tests do. dup_of TEXT, created_at TEXT NOT NULL, - salary_text TEXT NOT NULL DEFAULT '' + salary_text TEXT NOT NULL DEFAULT '', + teaser TEXT NOT NULL DEFAULT '', + seniority_model TEXT NOT NULL DEFAULT '', + seniority_stated TEXT NOT NULL DEFAULT '', + seniority_title TEXT NOT NULL DEFAULT '' +); +-- roles.teaser is the short text a list carries, kept until the full text +-- arrives and embedded while there is none; a later empty teaser does not +-- wipe it. seniority_model, seniority_stated and seniority_title are the +-- three levels behind roles.seniority: the model's (extract), the host's own +-- word mapped to app.seniority's vocabulary, and the title's. No comment sits +-- among the last columns: SQLite cannot drop a column a comment precedes, +-- which the migration tests do. + +-- The gazetteer: a place is a continent, a country (key the ISO code), a +-- region or a city, in the place parent_id names. key is unique per kind: +-- a region's key (app.places.REGIONS), a town's group. Seeded from the +-- dictionaries in app/models.py and app/places.py (app.places.seed). +CREATE TABLE IF NOT EXISTS places ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')), + key TEXT NOT NULL, + name TEXT NOT NULL, + parent_id INTEGER REFERENCES places(id), + UNIQUE (kind, key) +); + +-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn +-- and copenhagen are Copenhagen. One fold may name several places +-- (Luxembourg the town and the country). +CREATE TABLE IF NOT EXISTS place_aliases ( + fold TEXT NOT NULL, + place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, + UNIQUE (fold, place_id) +); + +-- What a place holds beyond its children: Øresund holds the Capital Region +-- of Denmark and Skåne, which lie in two countries. +CREATE TABLE IF NOT EXISTS place_includes ( + place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, + part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, + PRIMARY KEY (place_id, part_id) +); + +-- One row per place of a role (app/storage/gazetteer.py): the host's text in +-- the host's order (position), its fold, and the place it resolves to, NULL +-- while the gazetteer does not know it. A text naming several places is a +-- row for each. url is the link of a copy folded into this role, '' for +-- the role's own places. +CREATE TABLE IF NOT EXISTS role_places ( + uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, + position INTEGER NOT NULL, + text TEXT NOT NULL, + fold TEXT NOT NULL, + place_id INTEGER REFERENCES places(id), + url TEXT NOT NULL DEFAULT '' ); -- The open vocabulary summarise reads off a posting: the role family, the @@ -303,6 +361,11 @@ CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag); -- Partial: all but about 3% of roles are no copy. CREATE INDEX IF NOT EXISTS idx_roles_dup_of ON roles(dup_of) WHERE dup_of IS NOT NULL; +-- A role's places, and the roles in a set of places (the Place filter and +-- the country and place gates, `uid IN (... WHERE place_id IN ...)`). +CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid); +CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid); +CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id); """ #: The table whose absence marks a file that has never been a roledesk @@ -469,6 +532,10 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; # A Python step: the first rebuild runs the rule in app/dupes.py. (18, "roles.dup_of: one posting listed several times is one role, the others " "copies of it", lambda conn: _add_dup_of(conn)), + # A Python step: the seed and the resolver are Python. + (19, "places, place_aliases, place_includes and role_places: a role has a list of " + "places, resolved against a gazetteer; roles.teaser; the three seniority " + "sources", lambda conn: _add_places(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -592,6 +659,81 @@ def _add_dup_of(conn: sqlite3.Connection) -> None: log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups) +_V19 = """ +ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT ''; +ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT ''; +ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT ''; +ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT ''; +CREATE TABLE places ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')), + key TEXT NOT NULL, + name TEXT NOT NULL, + parent_id INTEGER REFERENCES places(id), + UNIQUE (kind, key) +); +CREATE TABLE place_aliases ( + fold TEXT NOT NULL, + place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, + UNIQUE (fold, place_id) +); +CREATE TABLE place_includes ( + place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, + part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, + PRIMARY KEY (place_id, part_id) +); +CREATE TABLE role_places ( + uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, + position INTEGER NOT NULL, + text TEXT NOT NULL, + fold TEXT NOT NULL, + place_id INTEGER REFERENCES places(id), + url TEXT NOT NULL DEFAULT '' +); +""" + + +def _add_places(conn: sqlite3.Connection) -> None: + """Migration 19's step. The tables and columns, the gazetteer seeded, and + every role's places: the `places` of its `raw` where a reader stated + them, else its `place` column cut at `;` (not `raw`'s, which a pull that + read the role's own page sends empty), each resolved once per distinct + text. + + The level splits into its sources without changing: on a role extract + has read, `seniority` is the model's; on any other, it is the stated + level when `raw` states one in the vocabulary, else the title's.""" + for statement in _statements(_V19): + conn.execute(statement) + gazetteer.seed(conn) + gaz = gazetteer.load(conn) + rows = conn.execute( + "SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall() + by_uid: list[tuple[str, list[str]]] = [] + levels = [] + for uid, raw, place, title, level, extracted_at in rows: + try: + given = json.loads(raw or "{}") + except ValueError: + given = {} + given = given if isinstance(given, dict) else {} + if found := gazetteer.texts({"places": given.get("places"), "place": place}): + by_uid.append((uid, found)) + stated = str(given.get("seniority") or "") + stated = stated if stated in SENIORITIES else "" + if extracted_at is not None: + levels.append((level, stated, from_title(title) if level else "", uid)) + elif stated: + levels.append(("", stated, from_title(title), uid)) + elif level: + levels.append(("", "", level, uid)) + written = gazetteer.write_many(conn, gaz, by_uid) + conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, " + "seniority_title = ? WHERE uid = ?", levels) + log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d", + len(gaz.places), written, len(by_uid), len(levels)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) @@ -659,6 +801,13 @@ def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None: conn.execute("COMMIT") +def _create(conn: sqlite3.Connection) -> None: + """A new file: `SCHEMA`, and the gazetteer seeded.""" + for statement in _statements(SCHEMA): + conn.execute(statement) + gazetteer.seed(conn) + + def initialise(conn: sqlite3.Connection) -> list[int]: """Create or upgrade, whichever this file needs. Returns migrations applied.""" fresh = conn.execute( @@ -668,7 +817,7 @@ def initialise(conn: sqlite3.Connection) -> list[int]: if fresh: # Atomic like a migration: a file half created would otherwise carry # the marker table and be taken for one to migrate. - _apply(conn, LATEST, SCHEMA) + _apply(conn, LATEST, _create) else: applied = migrate(conn) for statement in _statements(INDEXES): diff --git a/app/storage/store.py b/app/storage/store.py index dad2789..4f8d184 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -22,14 +22,14 @@ import numpy as np from desk.models import Item from app import dupes, places from app.models import ( - AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application, - ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow, - ScoredRole, + AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application, + ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace, + RoleRow, ScoredRole, Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key, ) from app.seniority import from_title -from app.storage import schema +from app.storage import gazetteer, schema #: Text columns a plugin may fill from `Item.raw` under the same name. An #: empty value never overwrites a filled column, so an upsert from a board that @@ -101,7 +101,8 @@ _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?" #: one, else the description's opening. A `SUBSTR` of the source column, not #: the column itself, so a big `description` never leaves SQLite whole for a #: row that only ever shows `EXCERPT_LIMIT` characters of it. -_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)" +_EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), " + "roles.teaser)") #: How many raw characters `roles_page` pulls before `app.models.excerpt` #: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim @@ -177,23 +178,34 @@ def unpack(blob: bytes) -> np.ndarray: return np.frombuffer(blob, dtype=_DTYPE) -def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]: +#: A role with no place the gazetteer resolved, over `roles r`. +_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)" + + +def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: """One gate as `(empty, match, params)`: SQL that is true when the role's column is empty, and SQL that is true when it matches, over `roles r`. - `params` belong to `match`; `empty` takes none.""" + `params` belong to `match`; `empty` takes none. + + `country` and `place` match over the gazetteer, as the feed's Place + filter does (`_places_sql`): a country gate is met by a role whose + `country` is one of its codes or that has a place in one of them, so + `DK` is met by `Copenhagen; London`. Its column is empty only when the + role has neither. A place gate's value is a place token, bare text a + `place:`.""" column = f"r.{gate.field}" values = [str(v).strip().lower() for v in gate.values if str(v).strip()] marks = ", ".join("?" for _ in values) or "NULL" if gate.field == "salary_min": floor = min(_int_or_none(v) or 0 for v in gate.values) return f"{column} IS NULL", f"{column} >= ?", [floor] + if gate.field == "country": + match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r", + codes=[v.upper() for v in values]) + return f"(r.country = '' AND {_UNPLACED})", match, params if gate.field == "place": - # Whole folded words, as the feed's Place filter: `lund` is not in - # `Billund`. A value is a place token, bare text a `place:`. - clauses = [places.match_sql(v, place="r.place_fold", country="r.country") - for v in values] - match = " OR ".join(sql for sql, _ in clauses) or "0" - return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps] + match, params = _places_sql(values, gaz, table="r") + return "r.place = ''", match, params if gate.field == "company": keys = [company_key(v) for v in values] return ("r.company_key = ''", @@ -205,18 +217,54 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]: return f"{column} = ''", f"LOWER({column}) IN ({marks})", values -def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]: - """The Place filter over `roles`: any of the tokens, each a `place_match` - call (`places.match_sql`). ValueError for a token that names nothing, - which would otherwise match no role without saying why.""" - clauses, params = [], [] +def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles", + codes: Sequence[str] = ()) -> tuple[str, list[Any]]: + """Any of the place tokens over `table`, as a join over the gazetteer: a + role with a place under one of the places the tokens name (`country:DK` + every place in Denmark, `region:skane` every town in Skåne, `place:malmo` + Malmö by any spelling), or whose `country` is one of their countries (a + role whose board gave a country and no place). `codes` are more values of + `country` to accept. A `place:` token the gazetteer does not know is + matched on whole words of the role's folded line (`places.match_sql`), + so a town nobody has entered yet is still found.""" + ids: set[int] = set() + found = {c for c in codes if c} + clauses: list[str] = [] + params: list[Any] = [] + for token in tokens: + named = gaz.lookup(token) + if named is None: + sql, values = places.match_sql(token, place=f"{table}.place_fold", + country=f"{table}.country") + clauses.append(sql) + params += values + continue + under = gaz.under(named) + ids |= under + found |= gaz.codes(under) + if ids: + clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places " + "WHERE place_id IN (SELECT value FROM json_each(?)))") + params.insert(0, json.dumps(sorted(ids))) + if found: + clauses.insert(1 if ids else 0, + f"{table}.country IN (SELECT value FROM json_each(?))") + params.insert(1 if ids else 0, json.dumps(sorted(found))) + return f"({' OR '.join(clauses) or '0'})", params + + +def _known_places(tokens: Sequence[str]) -> None: + """ValueError for a token that names nothing, which would otherwise match + no role without saying why.""" for token in tokens: if not places.known(token): raise ValueError(f"{token!r} is not a place this app knows") - sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country") - clauses.append(sql) - params += values - return f"({' OR '.join(clauses)})", params + + +def _text_rank(description: str, teaser: str) -> int: + """What a role is embedded from, as a rank: 2 its description, 1 its + teaser, 0 its title alone.""" + return 2 if description else 1 if teaser else 0 def _int_or_none(value: Any) -> int | None: @@ -351,6 +399,8 @@ class Store: self.path.parent.mkdir(parents=True, exist_ok=True) #: What "now" is. Replaced by tests that need two writes a second apart. self.clock: Callable[[], datetime] = lambda: datetime.now(UTC) + #: The gazetteer as last read, with the `gazetteer.VERSION` it was read at. + self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None with self.transaction() as conn: # Persistent: recorded in the file, inherited by every connection. conn.execute("PRAGMA journal_mode=WAL") @@ -364,7 +414,7 @@ class Store: `busy_timeout` matches `pipeline/table.py`, which writes the same file; `synchronous=NORMAL` is the standard WAL companion. `place_match` is registered on every one, since the place filter and the place gate - call it (`places.match_sql`).""" + call it for a town the gazetteer does not know (`_places_sql`).""" conn = sqlite3.connect(self.path) conn.row_factory = sqlite3.Row places.register(conn) @@ -400,6 +450,103 @@ class Store: def now(self) -> str: return _iso(self.clock()) + # ------------------------------------------------------------- gazetteer + + def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer: + """The gazetteer, read again only when its tables have grown, which + another process may have done: one cheap query a call.""" + version = tuple(conn.execute(gazetteer.VERSION).fetchone()) + if self._gaz is None or self._gaz[0] != version: + self._gaz = (version, gazetteer.load(conn)) + return self._gaz[1] + + def gazetteer(self) -> places.Gazetteer: + with closing(self._connect()) as conn: + return self._gazetteer(conn) + + def role_places(self, uid: str) -> list[RolePlace]: + """One role's places in the host's order, a copy's after the role's own.""" + return [RolePlace.from_row(r) for r in self._rows( + """SELECT rp.position, rp.text, rp.place_id, rp.url, + COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind + FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id + WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""", + (uid,))] + + def unresolved_places(self, open_only: bool = True, + limit: int | None = None) -> list[tuple[str, str, int]]: + """`(fold, text, roles)` for every place text no place of the + gazetteer resolves, most roles first: the list a person works through + with `add_place_alias` and `add_place`. `text` is one spelling of the + fold. A text that names no place ("Remote", `places.placeless`) is + left out; with `open_only`, so are closed roles and copies.""" + where = "rp.place_id IS NULL" + if open_only: + where += " AND r.closed_at IS NULL AND r.dup_of IS NULL" + rows = self._rows( + f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n + FROM role_places rp JOIN roles r ON r.uid = rp.uid + WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""") + found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows + if not places.placeless(r["fold"])] + return found[:limit] if limit is not None else found + + def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int: + """A place by id, or by a token that names exactly one place.""" + if isinstance(place, int): + if place not in gaz.places: + raise ValueError(f"there is no place {place}") + return place + named = gaz.lookup(place) or frozenset() + if len(named) != 1: + raise ValueError(f"{place!r} names {len(named)} places, not one") + return next(iter(named)) + + def add_place_alias(self, text: str, place: int | str) -> int: + """Write `text` as a spelling of `place` (an id, or a token such as + `place:copenhagen` or `country:DK` naming one place), and resolve again + every role with a place text that holds it. Returns how many roles now + resolve differently.""" + folded = places.fold(text) + if not folded: + raise ValueError("an alias needs a letter or a digit") + with self.transaction() as conn: + place_id = self._place_id(self._gazetteer(conn), place) + conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)", + (folded, place_id)) + return gazetteer.relink(conn, self._gazetteer(conn), folded) + + def add_place(self, name: str, kind: str, parent: int | str, + aliases: Sequence[str] = ()) -> int: + """Add a city or a region the gazetteer lacks, in `parent` (an id or a + token), written as `name` and each of `aliases`, and resolve again the + roles with a place text that holds one of them. Returns its id.""" + if kind not in ("city", "region"): + raise ValueError(f"a place added by hand is a city or a region, not {kind!r}") + folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f)) + if not folds: + raise ValueError("a place needs a name") + with self.transaction() as conn: + parent_id = self._place_id(self._gazetteer(conn), parent) + if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?", + (kind, folds[0])).fetchone(): + raise ValueError(f"there is already a {kind} {name!r}") + place_id = int(conn.execute( + "INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)", + (kind, folds[0], name.strip(), parent_id)).lastrowid or 0) + conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)", + [(f, place_id) for f in folds]) + gaz = self._gazetteer(conn) + for folded in folds: + gazetteer.relink(conn, gaz, folded) + return place_id + + def relink_places(self) -> int: + """Resolve every role's own places again, after the gazetteer was + seeded anew. Returns how many roles now resolve differently.""" + with self.transaction() as conn: + return gazetteer.relink(conn, self._gazetteer(conn)) + # ---------------------------------------------------------------- roles def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool: @@ -426,19 +573,32 @@ class Store: alone for good, whatever the plugin's raw guess now says — see `set_extracted` for the other half of the rule. - `seniority` is the title's level (`seniority.from_title`) until - extract has read the role and extract's from then on, as `function` - is. The classifier is called here rather than in a plugin, so a role - from any source gets it, the way `country` comes from `place` here - too. A level the board states in `raw` is a board field and wins over - both. `place_fold` is written with `place`, under the same keep-filled - rule. + `seniority` is the first filled of three levels. `seniority_model` + is extract's (`set_extracted`) and never written here. + `seniority_stated` is the host's word as `raw["seniority"]` carries + it, already in `SENIORITIES` (anything else is ''), and + `seniority_title` is `seniority.from_title`; both are rewritten every + pull. The classifier is called here rather than in a plugin, so a + role from any source gets it, the way `country` comes from `place`. + + The role's places (`role_places`) are the texts of `raw["places"]`, + else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved + against the gazetteer; `place` is the line shown, the texts joined + when `raw` has none. Like `place`, a pull with no place leaves them + as they are. `place_fold` is written with `place`, under the same + keep-filled rule. `teaser` is kept-filled too. + + A role is embedded from its description, else its teaser, else its + title (`app.jobs.embed.document`). When a pull brings a better one of + these than the role had, its vector is dropped so the next `embed` + reads it. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") raw = item.raw or {} company = str(raw.get("company") or (subscription.company if subscription else "")) - place = str(raw.get("place") or "") + place_texts = gazetteer.texts(raw) + place = str(raw.get("place") or "") or "; ".join(place_texts) stamp = self.now() published = _iso(item.published_at) row: dict[str, Any] = { @@ -463,12 +623,16 @@ class Store: "created_at": stamp, } row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT} + row["place"] = place row["country"] = row["country"] or country_of(place) row["remote"] = row["remote"] or remote_of(place) row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} row["function"] = str(raw.get("function") or "") stated = str(raw.get("seniority") or "") - row["seniority"] = stated or from_title(item.title) + row["seniority_stated"] = stated if stated in SENIORITIES else "" + row["seniority_title"] = from_title(item.title) + row["seniority"] = row["seniority_stated"] or row["seniority_title"] + row["teaser"] = str(raw.get("teaser") or "") row["place_fold"] = places.fold(row["place"]) # A board's own pay line, one per pull like title or published_at — # not kept-filled like _RAW_TEXT: a board that stops stating pay @@ -476,13 +640,13 @@ class Store: row["salary_text"] = str(raw.get("salary_text") or "") keep_filled = ("company", "company_key", "place_key", "remote", "description", - "feed_text", "original_url", *_RAW_TEXT) + "feed_text", "original_url", "teaser", *_RAW_TEXT) updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled] updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})" for c in (*_RAW_INT, "subscription_id")] updates += [f"{c}=excluded.{c}" for c in ("title", "title_key", "published_at", "raw", "last_seen", - "salary_text")] + "salary_text", "seniority_stated", "seniority_title")] updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)") # Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh # pull's classifier guess overwrite what extract decided, which is @@ -493,27 +657,29 @@ class Store: updates.append("function=CASE WHEN roles.extracted_at IS NULL " "THEN COALESCE(NULLIF(excluded.function, ''), roles.function) " "ELSE roles.function END") - # The title's level follows the title until extract has read the - # role, and is extract's from then on; a board's stated level wins. - updates.append("seniority=excluded.seniority" if stated else - "seniority=CASE WHEN roles.extracted_at IS NULL " - "THEN excluded.seniority ELSE roles.seniority END") + # The model's level, else the host's, else the title's. + updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), " + "NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)") updates.append("place_fold=CASE WHEN excluded.place != '' " "THEN excluded.place_fold ELSE roles.place_fold END") updates.append("closed_at=NULL") columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) with self.transaction() as conn: - before = conn.execute("SELECT description FROM roles WHERE uid=?", + before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?", (row["uid"],)).fetchone() conn.execute( f"INSERT INTO roles ({columns}) VALUES ({marks}) " f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row) - # A role listed without its text is embedded from its title, and - # `embed` encodes a role once. When the text arrives on a later - # pull, as it does for a host that states it on the role's own - # page, the vector goes, so the next `embed` reads the text. - if before is not None and not before["description"] and row["description"]: + if place_texts: + gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts) + # `embed` encodes a role once, from the best text it has. When a + # later pull brings a better one, as a host that states the text + # on the role's own page does, the vector goes, so the next + # `embed` reads it. + if before is not None and _text_rank(row["description"] or before["description"], + row["teaser"] or before["teaser"]) \ + > _text_rank(before["description"], before["teaser"]): conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],)) return before is None @@ -533,6 +699,7 @@ class Store: role = ScoredRole.from_row(row) role.tags = self._tags_of(uid) role.copies = self._copies_of(uid) + role.places = self.role_places(uid) return role def _copies_of(self, uid: str) -> list[RoleCopy]: @@ -685,7 +852,7 @@ class Store: `company_keys` keeps those exact companies. `places` keeps a role any of the place tokens finds (`app.places`: `country:DK`, `region:skane`, - `place:malmo`), matched on whole folded words; a token that names + `place:malmo`), over the gazetteer (`_places_sql`); a token that names nothing is a ValueError. `seniority` keeps the roles at any of those levels (`LEVELS`) and, with `seniority_unknown` (the default), those with no level, which is most of them. An empty list is no filter. @@ -934,8 +1101,8 @@ class Store: else: raise ValueError(f"tag_mode is any or all, not {tag_mode!r}") if places: - # Last, so the cheaper terms have cut the rows it is called on. - clause, values = _places_sql(places) + _known_places(places) + clause, values = _places_sql(places, self.gazetteer()) where.append(clause) params += values sql = f"FROM roles {_WITH_VOTES}" @@ -1087,9 +1254,9 @@ class Store: non-empty answer overwrites it outright rather than only filling an empty column — see `upsert_role`'s docstring for the other half of the rule that then keeps a later pull from putting the guess back. - `seniority` is the other: `upsert_role` fills it from the title, so a - non-empty answer replaces it, unless the board stated a level in - `raw`.""" + `seniority` is the other: a non-empty answer is `seniority_model`, + which comes before the stated and the title's level, so it becomes + `seniority` too.""" with self.transaction() as conn: row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone() if row is None: @@ -1103,8 +1270,8 @@ class Store: if function: sets["function"] = function level = str(fields.get("seniority") or "") - if level and not json.loads(row["raw"] or "{}").get("seniority"): - sets["seniority"] = level + if level: + sets["seniority_model"] = sets["seniority"] = level for name in _RAW_INT: number = _int_or_none(fields.get(name)) if number is not None and row[name] is None: @@ -1112,10 +1279,12 @@ class Store: languages = [str(code) for code in fields.get("languages") or []] if languages and json.loads(row["languages"] or "[]") == []: sets["languages"] = json.dumps(languages) - filled = len(sets) + filled = len(sets) - ("seniority" in sets) if "place" in sets: sets["place_key"] = place_key(sets["place"]) sets["place_fold"] = places.fold(sets["place"]) + gazetteer.write(conn, self._gazetteer(conn), uid, + gazetteer.texts({"place": sets["place"]})) sets["extracted_at"] = self.now() conn.execute( f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?", @@ -1375,8 +1544,9 @@ class Store: as a column so a gated role keeps its row and reason.""" columns: list[str] = [] params: list[Any] = [] + gaz = self.gazetteer() for n, gate in enumerate(gates): - empty, match, values = _gate_sql(gate) + empty, match, values = _gate_sql(gate, gaz) columns.append( f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}") params += values diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py index 9dfbc75..1c8cbc1 100644 --- a/tests/test_dupes_storage.py +++ b/tests/test_dupes_storage.py @@ -164,13 +164,17 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None: store.upsert_role(item) store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))]) with sqlite3.connect(path) as conn: + for table in ("role_places", "place_includes", "place_aliases", "places"): + conn.execute(f"DROP TABLE {table}") + for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"): + conn.execute(f"ALTER TABLE roles DROP COLUMN {column}") conn.execute("DROP INDEX idx_roles_dup_of") conn.execute("ALTER TABLE roles DROP COLUMN dup_of") conn.execute("PRAGMA user_version = 17") migrated = Store(path) - assert migrated.migrations_applied == [18] + assert migrated.migrations_applied == [18, 19] assert dup_of(migrated) == {first.uid: None, second.uid: first.uid} with sqlite3.connect(path) as conn: assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'" diff --git a/tests/test_gazetteer.py b/tests/test_gazetteer.py new file mode 100644 index 0000000..4e5e9f7 --- /dev/null +++ b/tests/test_gazetteer.py @@ -0,0 +1,397 @@ +"""The gazetteer: its seed, the resolver, and a role's places in the store. + +The store tests go through `Store`, as the feed and the gates do. The +real-line test compares the join over the gazetteer with the whole-word rule +it replaced (`places.matches`) on the live lines `test_places` holds: the +join must find every role the word rule found, but for the lines listed where +the word rule was wrong. +""" + +from __future__ import annotations + +import sqlite3 +from collections import Counter +from collections.abc import Iterator +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from desk.models import Item +from app import places +from app.models import Gate, _CITIES, _COUNTRY_NAMES +from app.places import Gazetteer, fold, seed +from app.storage import Store + +from test_places import LINES, ORACLE_TOKENS + + +def gazetteer() -> Gazetteer: + seeds, includes = seed() + ids: dict[tuple[str, str], int] = {} + rows, aliases = [], [] + for n, s in enumerate(seeds, 1): + ids[(s.kind, s.key)] = n + rows.append((n, s.kind, s.key, s.name, ids[s.parent] if s.parent else None)) + aliases += [(a, n) for a in s.aliases] + return Gazetteer(rows, aliases, [(ids[a], ids[b]) for a, b in includes]) + + +GAZ = gazetteer() + + +def names(text: str) -> list[str]: + return [GAZ.places[p].name for p in GAZ.resolve(text)] + + +# ---------------------------------------------------------------------- seed + +def test_the_seed_holds_the_dictionaries_in_code() -> None: + seeds, includes = seed() + kinds = Counter(s.kind for s in seeds) + assert kinds["continent"] == 1 + assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line) + keys = {(s.kind, s.key) for s in seeds} + assert {("region", key) for key in places.REGIONS} <= keys + assert {("region", "oresund"), ("region", "skane")} in [set(i) for i in includes] + folds = {a for s in seeds for a in s.aliases} + assert set(_CITIES) - {"region hovedstaden"} <= folds + assert {"kobenhavn", "kopenhamn", "copenhagen", "skane", "emea", "eu"} <= folds + + +def test_parents_come_before_their_children() -> None: + seen: set[tuple[str, str]] = set() + for s in seed()[0]: + assert s.parent is None or s.parent in seen, s + seen.add((s.kind, s.key)) + + +def test_copenhagen_is_in_the_capital_region_in_denmark_in_europe() -> None: + copenhagen = GAZ.by_key[("city", "copenhagen")] + above = {GAZ.places[p].key for p in GAZ.ancestors(copenhagen)} + assert above == {"hovedstaden", "oresund", "DK", "europe"} + assert GAZ.country(copenhagen) == "DK" + assert GAZ.by_key[("country", "US")] not in GAZ.under({GAZ.by_key[("continent", "europe")]}) + + +# ------------------------------------------------------------------ resolving + +@pytest.mark.parametrize("text, expected", [ + ("Aarhus N, Region Central Jutland, Denmark", ["Aarhus"]), + ("København K", ["Copenhagen"]), + ("Copenhagen Office", ["Copenhagen"]), + ("Malmö/Lund", ["Malmö", "Lund"]), + ("Aalborg, Copenhagen, Kolding", ["Aalborg", "Copenhagen", "Kolding"]), + ("Paderborn, Germany", ["Germany"]), + ("Skåne län, Sweden", ["Skåne"]), + ("London, Ontario", ["Ontario"]), + ("Bergen op Zoom, Netherlands", ["Netherlands"]), + ("Mexicali, BAJA CALIFORNIA, Mexico", ["Mexico"]), + ("Paris, TX", ["United States"]), + ("Remote (DE)", ["Germany"]), + ("London OR Paris OR Germany", ["London", "Paris", "Germany"]), + ("Allerød or Stockholm (SE)", ["Allerød", "Stockholm"]), + ("Finland, København, Litauen, Polen, Sverige", ["Finland", "Copenhagen", "Sweden"]), + ("Remote, EU or US", ["Europe", "United States"]), + ("Home based - EMEA", ["Europe"]), + ("Luxembourg", ["Luxembourg"]), + ("Nordhavn og mulighed for hjemmearbejde", ["Nordhavn"]), + ("Remote", []), + ("Oberkochen", []), + ("", []), +]) +def test_a_text_resolves_to_the_places_it_names(text: str, expected: list[str]) -> None: + assert names(text) == expected + + +def test_luxembourg_the_town_is_in_luxembourg_the_country() -> None: + town = GAZ.by_key[("city", "luxembourg")] + assert GAZ.resolve("Luxembourg") == (town,) + assert GAZ.country(town) == "LU" + + +@pytest.mark.parametrize("text", ["Remote", "Hybrid", "Multiple Locations", "2 Locations", + "Remote job", "", " "]) +def test_a_text_of_how_a_role_is_worked_is_no_place(text: str) -> None: + assert places.placeless(text) + + +@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Jena"]) +def test_a_text_with_a_name_is_a_place(text: str) -> None: + assert not places.placeless(text) + + +# -------------------------------------------------------------------- tokens + +def test_a_token_names_places_before_what_lies_in_them() -> None: + assert GAZ.lookup("country:DK") == {GAZ.by_key[("country", "DK")]} + assert GAZ.lookup("country:EU") == {GAZ.by_key[("continent", "europe")]} + assert GAZ.lookup("region:Skåne") == {GAZ.by_key[("region", "skane")]} + assert GAZ.lookup("place:kobenhavn") == {GAZ.by_key[("city", "copenhagen")]} + assert GAZ.lookup("place:denmark") == {GAZ.by_key[("country", "DK")]} + assert GAZ.lookup("place:oberkochen") is None + assert GAZ.lookup("country:ZZ") is None + + +def test_europe_is_every_european_country_and_eu() -> None: + under = GAZ.under(GAZ.lookup("country:EU") or ()) + codes = GAZ.codes(under) + assert {"DK", "SE", "DE", "GB", "CH", "EU"} <= codes + assert not {"US", "CN"} & codes + + +def test_oresund_holds_both_capital_regions() -> None: + under = GAZ.under(GAZ.lookup("region:oresund") or ()) + assert {GAZ.by_key[("city", "malmo")], GAZ.by_key[("city", "copenhagen")]} <= under + assert GAZ.by_key[("city", "aarhus")] not in under + + +# --------------------------------------------------------------------- store + +@pytest.fixture +def store(tmp_path: Path) -> Iterator[Store]: + yield Store(tmp_path / "roledesk.db") + + +def posting(source_id: str, *, title: str = "Backend Engineer", body: str = "", + **raw: object) -> Item: + item = Item(source_id=source_id, location=f"https://boards.example/{source_id}", + title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC), + raw=dict(raw)) + item.source = "ats" + return item + + +def texts(store: Store, uid: str) -> list[tuple[int, str, str]]: + return [(p.position, p.text, p.name) for p in store.role_places(uid)] + + +def test_a_fresh_file_is_seeded(store: Store) -> None: + gaz = store.gazetteer() + assert len(gaz.places) == len(GAZ.places) + assert ("city", "copenhagen") in gaz.by_key + + +def test_a_joined_line_is_several_places(store: Store) -> None: + store.upsert_role(posting("1", place="Copenhagen; London; Warsaw")) + uid = posting("1").uid + assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"), + (2, "Warsaw", "Warsaw")] + role = store.role(uid) + assert role is not None and role.place == "Copenhagen; London; Warsaw" + assert [p.name for p in role.places] == ["Copenhagen", "London", "Warsaw"] + + +def test_the_places_of_the_reader_contract_come_first(store: Store) -> None: + store.upsert_role(posting("1", place="Berlin, Munich", places=[ + {"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False}, + {"text": "Remote (DE)", "remote": True}])) + assert texts(store, posting("1").uid) == [ + (0, "Berlin", "Berlin"), (1, "Munich", "Munich"), (2, "Remote (DE)", "Germany")] + + +def test_places_without_a_line_make_the_line(store: Store) -> None: + store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None}, + {"text": "Malmö", "remote": None}])) + role = store.role(posting("1").uid) + assert role is not None + assert (role.place, role.country) == ("Lund; Malmö", "SE") + + +def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None: + store.upsert_role(posting("1", place="Aarhus")) + store.upsert_role(posting("1")) + assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")] + store.upsert_role(posting("1", place="Odense")) + assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")] + + +def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None: + store.upsert_role(posting("1", place="Aarhus")) + uid = posting("1").uid + with sqlite3.connect(store.path) as conn: + conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) " + "VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,)) + store.upsert_role(posting("1", place="Aalborg")) + assert [(p.text, p.url) for p in store.role_places(uid)] == [ + ("Aalborg", ""), ("Odense", "https://copy")] + + +def test_an_extracted_place_is_resolved(store: Store) -> None: + store.upsert_role(posting("1")) + uid = posting("1").uid + store.set_extracted(uid, {"place": "Lund, Sweden"}) + assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")] + + +# ---------------------------------------------------------- filter and gates + +def uids(store: Store, *tokens: str) -> set[str]: + return {r.uid for r in store.roles(limit=100_000, places=list(tokens), by_score=False)} + + +@pytest.fixture +def several(store: Store) -> dict[str, str]: + lines = {"cph": "Copenhagen; London", "eu": "Berlin, Germany; Paris, France", + "malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund", + "zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"} + for key, line in lines.items(): + store.upsert_role(posting(key, place=line)) + return {key: posting(key).uid for key in lines} + + +def test_a_country_is_every_place_in_it(store: Store, several: dict[str, str]) -> None: + assert uids(store, "country:DK") == {several["cph"], several["billund"]} + assert uids(store, "country:SE") == {several["malmo"], several["lund"], several["remote"]} + assert uids(store, "country:DE") == {several["eu"]} + assert uids(store, "country:GB") == {several["cph"]} + + +def test_europe_is_every_european_role(store: Store, several: dict[str, str]) -> None: + assert uids(store, "country:EU") == set(several.values()) - {several["ny"], several["zeiss"]} + + +def test_a_region_is_every_town_in_it(store: Store, several: dict[str, str]) -> None: + assert uids(store, "region:skane") == {several["malmo"], several["lund"]} + assert uids(store, "region:oresund") == {several["malmo"], several["lund"], several["cph"]} + assert uids(store, "place:skane") == uids(store, "region:skane") + + +def test_a_town_is_its_place_and_lund_is_not_in_billund(store: Store, + several: dict[str, str]) -> None: + assert uids(store, "place:lund") == {several["lund"]} + assert uids(store, "place:kobenhavn") == {several["cph"]} + + +def test_a_town_the_gazetteer_lacks_is_found_by_its_words(store: Store, + several: dict[str, str]) -> None: + assert uids(store, "place:oberkochen") == {several["zeiss"]} + + +def test_a_token_that_names_nothing_is_refused(store: Store) -> None: + with pytest.raises(ValueError): + store.roles(places=["region:atlantis"]) + + +def test_a_country_given_without_a_place_still_counts(store: Store) -> None: + store.upsert_role(posting("1", country="DK")) + assert uids(store, "country:DK") == {posting("1").uid} + + +def gates(store: Store, gate: Gate) -> dict[str, int]: + agent = store.save_agent("A", gates=[gate]) + return {c.uid: c.gates[0] for c in store.score_candidates(agent.id, agent.gates)} + + +def test_a_country_gate_is_met_by_any_place_in_the_country(store: Store, + several: dict[str, str]) -> None: + found = gates(store, Gate(field="country", values=["DK", "SE"], mode="any")) + assert {uid for uid, g in found.items() if g == 1} == { + several[k] for k in ("cph", "billund", "malmo", "lund", "remote")} + # Only a role with neither a country nor a known place has no evidence. + assert {uid for uid, g in found.items() if g == -1} == {several["zeiss"]} + assert found[several["ny"]] == 0 and found[several["eu"]] == 0 + + +def test_the_eu_value_of_a_country_gate_is_europe(store: Store, + several: dict[str, str]) -> None: + found = gates(store, Gate(field="country", values=["EU"])) + assert found[several["eu"]] == 1 and found[several["cph"]] == 1 + assert found[several["ny"]] == 0 + + +def test_a_place_gate_matches_over_the_gazetteer(store: Store, + several: dict[str, str]) -> None: + found = gates(store, Gate(field="place", values=["Malmö", "Lund", "oberkochen"], + mode="any")) + assert {uid for uid, g in found.items() if g == 1} == { + several["malmo"], several["lund"], several["zeiss"]} + + +# ------------------------------------------------------------ unresolved texts + +def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None: + for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")): + store.upsert_role(posting(str(n), place=line)) + assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1), + ("jena", "Jena", 1)] + assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)] + + +def test_an_alias_links_what_it_names(store: Store) -> None: + store.upsert_role(posting("1", place="Kbh. K")) + store.upsert_role(posting("2", place="Copenhagen")) + uid = posting("1").uid + assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)] + assert store.add_place_alias("Kbh", "place:copenhagen") == 1 + assert texts(store, uid) == [(0, "Kbh. K", "Copenhagen")] + assert store.unresolved_places() == [] + assert uids(store, "country:DK") == {uid, posting("2").uid} + # The second store, another process, sees the new alias too. + assert Store(store.path).gazetteer().lookup("place:kbh") == \ + store.gazetteer().lookup("place:copenhagen") + + +def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None: + store.upsert_role(posting("1", place="Oberkochen; Jena")) + store.upsert_role(posting("2", place="Paderborn, Germany")) + oberkochen = store.add_place("Oberkochen", "city", "country:DE") + store.add_place("Paderborn", "city", "country:DE") + assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")] + assert texts(store, posting("2").uid) == [(0, "Paderborn, Germany", "Paderborn")] + assert uids(store, "place:oberkochen") == {posting("1").uid} + assert store.gazetteer().country(oberkochen) == "DE" + with pytest.raises(ValueError): + store.add_place("Oberkochen", "city", "country:DE") + with pytest.raises(ValueError): + store.add_place("Atlantis", "country", "country:EU") + + +def test_an_alias_for_no_single_place_is_refused(store: Store) -> None: + with pytest.raises(ValueError): + store.add_place_alias("Kbh", "place:atlantis") + with pytest.raises(ValueError): + store.add_place_alias("!!", "place:copenhagen") + + +def test_relink_places_resolves_every_role_again(store: Store) -> None: + store.upsert_role(posting("1", place="Oberkochen")) + with sqlite3.connect(store.path) as conn: + conn.execute("UPDATE role_places SET place_id = " + "(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')") + assert store.relink_places() == 1 + assert texts(store, posting("1").uid) == [(0, "Oberkochen", "")] + + +# --------------------------------------------------------- against the words + +#: Lines of `test_places.LINES` the word rule found for a token and the +#: gazetteer does not, each rightly: the London in Canada, and a town named +#: after Zurich in Illinois. +NOT_FOUND_ANY_MORE: dict[str, set[str]] = { + "country:GB": {"London, Canada; Remote, United States; Oak Brook, United States"}, + "place:london": {"London, Canada; Remote, United States; Oak Brook, United States"}, + "place:zurich": {"Lake Zurich, Illinois, United States"}, +} + + +@pytest.fixture(scope="module") +def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tuple[str, str]]]: + store = Store(tmp_path_factory.mktemp("real") / "roledesk.db") + lines = {} + for n, (place, country) in enumerate(LINES): + item = posting(f"line{n}", place=place, country=country) + store.upsert_role(item) + lines[item.uid] = (place, country) + return store, lines + + +@pytest.mark.parametrize("token", ORACLE_TOKENS) +def test_the_gazetteer_finds_what_the_words_found( + real: tuple[Store, dict[str, tuple[str, str]]], token: str) -> None: + store, lines = real + by_words = {uid for uid, (place, country) in lines.items() + if places.matches(fold(place), country, token)} + by_places = uids(store, token) + lost = {lines[uid][0] for uid in by_words - by_places} + assert lost <= NOT_FOUND_ANY_MORE.get(token, set()), (token, lost) diff --git a/tests/test_storage.py b/tests/test_storage.py index 9c08464..9d125a4 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -14,8 +14,8 @@ from desk.models import Item import numpy as np from app.models import ( - LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key, - posted_at, posted_host, title_key, + LEVELS, Application, Gate, Limits, Prompt, company_key, place_key, + posted_at, posted_host, title_key, trusted_posted, ) from app.storage.store import pack from app.storage import LATEST, Store @@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 18 + assert LATEST == 19 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -88,8 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]: return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")] +def _undo_v19(conn: sqlite3.Connection) -> None: + """A fresh file put back to the v18 shape: no gazetteer, no role places, + no teaser, one level.""" + for table in ("role_places", "place_includes", "place_aliases", "places"): + conn.execute(f"DROP TABLE {table}") + for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"): + conn.execute(f"ALTER TABLE roles DROP COLUMN {column}") + + def _undo_v18(conn: sqlite3.Connection) -> None: """A fresh file put back to the v17 shape: no copies.""" + _undo_v19(conn) conn.execute("DROP INDEX idx_roles_dup_of") conn.execute("ALTER TABLE roles DROP COLUMN dup_of") @@ -166,7 +176,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated( migrated = Store(path) - assert migrated.migrations_applied == [15, 16, 17, 18] + assert migrated.migrations_applied == [15, 16, 17, 18, 19] acme = migrated.company("acme") assert acme is not None assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example") @@ -371,7 +381,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14, 15, 16, 17, 18] + assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -395,7 +405,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None migrated = Store(path) - assert migrated.migrations_applied == [16, 17, 18] + assert migrated.migrations_applied == [16, 17, 18, 19] acme = migrated.company("acme") assert acme is not None assert (acme.careers_url, acme.notes, acme.hand_reason) == ( @@ -424,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( migrated = Store(path) - assert migrated.migrations_applied == [17, 18] + assert migrated.migrations_applied == [17, 18, 19] with sqlite3.connect(path) as conn: rows = {uid: (fold, level) for uid, fold, level in conn.execute( "SELECT uid, place_fold, seniority FROM roles")} @@ -517,7 +527,7 @@ def test_posted_host_reads_board_then_provider() -> None: def test_posted_at_trusts_only_a_known_host() -> None: - assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS + assert trusted_posted("greenhouse") and not trusted_posted("taleo") early = "2020-01-01T00:00:00+00:00" late = "2026-09-23T09:00:00+00:00" # A trusted host's earlier date wins over first_seen... @@ -527,6 +537,29 @@ def test_posted_at_trusts_only_a_known_host() -> None: assert posted_at(early, late, "") == late +def test_a_reader_that_declares_reads_decides_its_own_date_trust( + monkeypatch: pytest.MonkeyPatch) -> None: + """`fields.posted` of a declared reader wins over the fallback set, either + way; a reader without `READS`, or a provider that is none, keeps it.""" + import app.models as models + + monkeypatch.setattr(models, "_declared_posted", + lambda: {"greenhouse": "listed", "gem": "trusted"}) + assert not trusted_posted("greenhouse") + assert trusted_posted("gem") + assert trusted_posted("jobindex") + assert not trusted_posted("taleo") + + +def test_the_declarations_are_read_off_the_readers() -> None: + import app.models as models + from plugins.readers import READERS + + declared = models._declared_posted() + assert set(declared) <= set(READERS) + assert all(value in ("trusted", "listed", "none") for value in declared.values()) + + def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None: first_seen = "2026-09-23T09:00:00+00:00" future = "2026-09-24T00:00:00+00:00" # after first_seen @@ -574,6 +607,92 @@ def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) assert store.roles_to_embed("m", 10) == [] +def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives( + store: Store) -> None: + from app.jobs.embed import document + + listed = posting(teaser="Build the payments core.") + listed.body = "" + bare = posting() + bare.body = "" + store.upsert_role(bare) # the title alone + store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))]) + store.upsert_role(listed) # a teaser beats the title + assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] + role = store.role(listed.uid) + assert role is not None and role.teaser == "Build the payments core." + assert document(role) == "Senior Backend Engineer (m/w/d)\nBuild the payments core." + assert store.roles_page()[0].excerpt == "Build the payments core." + + store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) + store.upsert_role(bare) # a pull without it keeps it + assert store.roles_to_embed("m", 10) == [] + assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr] + store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it + assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] + role = store.role(listed.uid) + assert role is not None and document(role).endswith("We build things.") + + store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) + store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text + assert store.roles_to_embed("m", 10) == [] + + +def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None: + store.upsert_role(posting("1")) # "Senior Backend Engineer" + uid = posting("1").uid + + def levels() -> tuple[str, str, str, str]: + role = store.role(uid) + assert role is not None + return (role.seniority, role.seniority_model, role.seniority_stated, + role.seniority_title) + + assert levels() == ("senior", "", "", "senior") + store.upsert_role(posting("1", seniority="lead")) + assert levels() == ("lead", "", "lead", "senior") + store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary + assert levels() == ("senior", "", "", "senior") + store.set_extracted(uid, {"seniority": "staff"}) + store.upsert_role(posting("1", seniority="lead")) + assert levels() == ("staff", "staff", "lead", "senior") + + +def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> None: + path = tmp_path / "v18.db" + store = Store(path) + joined = posting("joined", place="Copenhagen; Malmö") + detailed = posting("detailed", place="Aarhus") + stated = posting("stated", seniority="lead") + extracted = posting("extracted") + for item in (joined, detailed, stated, extracted): + store.upsert_role(item) + with sqlite3.connect(path) as conn: + _undo_v19(conn) + # A role whose own page was read is pulled with an empty place in raw. + conn.execute("UPDATE roles SET raw = '{}' WHERE uid = ?", (detailed.uid,)) + conn.execute("UPDATE roles SET extracted_at = '2026-09-23', seniority = 'staff' " + "WHERE uid = ?", (extracted.uid,)) + conn.execute("PRAGMA user_version = 18") + + migrated = Store(path) + + assert migrated.migrations_applied == [19] + assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"] + assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"] + assert migrated.role_places(stated.uid) == [] + with sqlite3.connect(path) as conn: + levels = {uid: rest for uid, *rest in conn.execute( + "SELECT uid, seniority, seniority_model, seniority_stated, seniority_title, teaser " + "FROM roles")} + assert levels == { + joined.uid: ["senior", "", "", "senior", ""], + detailed.uid: ["senior", "", "", "senior", ""], + stated.uid: ["lead", "", "lead", "senior", ""], + extracted.uid: ["staff", "staff", "", "senior", ""], + } + + def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000", salary_max=95000, currency="EUR", department="Eng")) @@ -738,12 +857,20 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N assert store.role(uid).seniority == "lead" # type: ignore[union-attr] -def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None: +def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both( + store: Store) -> None: store.upsert_role(posting("1", seniority="junior")) uid = posting("1").uid - assert store.role(uid).seniority == "junior" # type: ignore[union-attr] - store.set_extracted(uid, {"seniority": "senior"}) - assert store.role(uid).seniority == "junior" # type: ignore[union-attr] + role = store.role(uid) + assert role is not None + assert (role.seniority, role.seniority_stated, role.seniority_title, + role.seniority_model) == ("junior", "junior", "senior", "") + store.set_extracted(uid, {"seniority": "staff"}) + store.upsert_role(posting("1", seniority="junior")) + role = store.role(uid) + assert role is not None + assert (role.seniority, role.seniority_stated, role.seniority_model) == \ + ("staff", "junior", "staff") def test_company_falls_back_to_the_subscription(store: Store) -> None: