Places as a gazetteer, role_places, teaser and three seniority sources

Migration 19 adds places, place_aliases, place_includes (seeded from the
dictionaries in app/models.py and app/places.py, Europe as a continent over
the European countries) and role_places, one row per place of a role,
resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them
from raw["places"], else the place line cut at ";"; the migration builds them
from each role's place column.

The Place filter and the country and place gates are joins over the tree:
country:DK is any place under Denmark, region:skane any town in Skane, a
country gate is met by any place in its countries, and the EU fallback of
the word matcher is no longer used. A place: token the gazetteer does not
know still matches whole words. Store gains gazetteer, role_places,
unresolved_places, add_place_alias, add_place and relink_places.

roles.teaser is kept-filled and embedded while there is no description; a
vector is dropped when a better text arrives. seniority_model, _stated and
_title sit behind roles.seniority, which stays the first filled of them.

TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads
READS.fields.posted and falls back to the old set for undeclared readers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:31:45 +02:00
1 parent 4c0da58da7
commit bbd7522163
10 files changed
+1437 -98

No files matched your search

+73 -16
View File
@@ -12,7 +12,10 @@ from __future__ import annotations
import json
import re
import sys
import unicodedata
from functools import lru_cache
from pathlib import Path
from typing import Annotated, Any, Literal, get_args
from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator
@@ -358,8 +361,9 @@ _REGIONS = ("europe", "emea", "eu", "european union", "eea")
#: state or a Canadian province in the same part of the line beats one of
#: these ("London, Ontario" is Canada); a country name does not lose to it.
_CITIES: dict[str, str] = {
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
**dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck"), "AT"),
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege",
"luik"), "BE"),
**dict.fromkeys(("sofia",), "BG"),
**dict.fromkeys(("zagreb",), "HR"),
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
@@ -369,7 +373,7 @@ _CITIES: dict[str, str] = {
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
**dict.fromkeys(("tallinn", "tartu"), "EE"),
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
**dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"),
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
"lille", "rennes", "amiens"), "FR"),
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
@@ -378,27 +382,30 @@ _CITIES: dict[str, str] = {
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck"), "DE"),
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
**dict.fromkeys(("budapest",), "HU"),
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
"florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples",
"napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"),
**dict.fromkeys(("riga",), "LV"),
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
**dict.fromkeys(("luxembourg", "luxemburg"), "LU"),
**dict.fromkeys(("valletta",), "MT"),
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
"delft"), "NL"),
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
"PL"),
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"),
**dict.fromkeys(("belgrade", "beograd"), "RS"),
**dict.fromkeys(("bratislava",), "SK"),
**dict.fromkeys(("ljubljana",), "SI"),
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
"granada", "santiago de compostela"), "ES"),
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras",
"linkoping", "orebro", "norrkoping", "jonkoping"), "SE"),
**dict.fromkeys(("reykjavik",), "IS"),
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
@@ -408,7 +415,7 @@ _CITIES: dict[str, str] = {
"slough", "southampton", "swindon", "liverpool", "redhill",
"milton keynes"), "GB"),
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
"sankt gallen", "st gallen"), "CH"),
"sankt gallen", "st gallen", "lucerne", "luzern"), "CH"),
**dict.fromkeys(("kyiv", "kiev"), "UA"),
**dict.fromkeys(("istanbul", "ankara"), "TR"),
**dict.fromkeys(("tirana",), "AL"),
@@ -659,9 +666,10 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
# -------------------------------------------------------------- posting dates
#: Hosts whose date is a creation or first-publication timestamp, safe to
#: read as when a role was posted, checked against live boards on
#: 2026-09-29:
#: Hosts whose date is a creation or first-publication timestamp, for a
#: reader that does not declare `READS` yet (see `trusted_posted`); a reader
#: that declares it says so with `fields.posted="trusted"`. Checked against
#: live boards on 2026-09-29:
#:
#: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers
#: `first_published` over `updated_at`. A live board (adyen) showed a job
@@ -708,14 +716,39 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
#: HR Manager) state a date nobody checked, and Pinpoint states none.
#:
#: A host not listed here, or a role whose stored `published_at` fails the
#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the
#: safe default this migration is replacing everywhere else.
TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({
#: sanity check in `posted_at`, keeps `first_seen` as its posted date.
_UNDECLARED_TRUSTED: frozenset[str] = frozenset({
"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
"personio", "teamtailor", "jobindex", "adzuna", "platsbanken",
"workday", "successfactors", "oracle", "phenom", "bamboohr", "jobylon", "rippling",
})
@lru_cache(maxsize=1)
def _declared_posted() -> dict[str, str]:
"""`fields.posted` of every reader that declares `READS`, by name. Read
once: the readers are loaded when `plugins.readers` is imported. `plugins`
is a top-level folder beside `app`, reached as `app/jobs/extract.py`
reaches it."""
root = str(Path(__file__).resolve().parents[1])
if root not in sys.path:
sys.path.insert(0, root)
from plugins.readers import READERS
return {name: module.READS.fields.posted for name, module in READERS.items()
if getattr(module, "READS", None) is not None}
def trusted_posted(host: str) -> bool:
"""Whether `host` states a date that is safe to read as when a role was
posted: its reader declares `fields.posted="trusted"`, or, for a reader
without `READS` (or a search provider not yet a reader), it is in
`_UNDECLARED_TRUSTED`."""
declared = _declared_posted()
if host in declared:
return declared[host] == "trusted"
return host in _UNDECLARED_TRUSTED
#: A `published_at` before this is a parsing slip or a placeholder, not a
#: real posting date: no board here existed before 2015.
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
@@ -732,7 +765,7 @@ def posted_host(raw: dict[str, Any]) -> str:
def posted_at(published_at: str, first_seen: str, host: str) -> str:
"""The date a role counts as posted: the earlier of `first_seen` (when
the app itself first stored it) and `published_at` (what the board
claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and
claims), but only when `trusted_posted(host)` and
`published_at` is neither absurdly old nor later than `first_seen` — a
role cannot be posted after the app already saw it, trusted host or not,
which also catches a `published_at` the board reports in the future
@@ -740,7 +773,7 @@ def posted_at(published_at: str, first_seen: str, host: str) -> str:
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
is exact and needs no parsing. A role with no trusted date, or with one
that fails the check, keeps `first_seen`."""
if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen:
if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen:
return published_at
return first_seen
@@ -775,6 +808,20 @@ class RoleCopy(Model):
posted_at: str = ""
class RolePlace(Model):
"""One place of a role (`role_places`): the host's `text`, and the place
of the gazetteer it resolves to, `place_id` None (and `name` and `kind`
empty) while it resolves to none. `url` is the link of a copy folded into
the role, '' for the role's own places."""
position: int = 0
text: str = ""
place_id: int | None = None
name: str = ""
kind: str = ""
url: str = ""
class Role(Model):
"""A row of `roles`: one posting on one board."""
@@ -797,13 +844,20 @@ class Role(Model):
#: could be normalised into `salary_min`/`salary_max`/`currency`
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
salary_text: str = ""
#: The level shown and filtered on: the first filled of `seniority_model`,
#: `seniority_stated` and `seniority_title`.
seniority: str = ""
seniority_model: str = ""
seniority_stated: str = ""
seniority_title: str = ""
function: str = ""
permit: Permit = ""
#: ISO 639-1 codes the posting says are required, lowercase.
languages: JsonStrings = []
description: str = ""
feed_text: str = ""
#: The short text the host's list carries, until `description` arrives.
teaser: str = ""
original_url: str = ""
published_at: str
#: The date this role counts as posted: see `posted_at`. Used for
@@ -838,6 +892,9 @@ class Role(Model):
#: The open copies of this role, oldest first, when it is the kept one;
#: empty otherwise. Not a column: filled by `Store.role`.
copies: list[RoleCopy] = []
#: Every place of the role, in the host's order. Not a column: filled by
#: `Store.role` from `role_places`.
places: list[RolePlace] = []
@computed_field # type: ignore[prop-decorator]
@property