Places as a gazetteer, role_places, teaser and three seniority sources
Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
bbd7522163
10 files changed
+1437
-98
No files matched your search
+73
-16
@@ -12,7 +12,10 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, Literal, get_args
|
||||
|
||||
from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator
|
||||
@@ -358,8 +361,9 @@ _REGIONS = ("europe", "emea", "eu", "european union", "eea")
|
||||
#: state or a Canadian province in the same part of the line beats one of
|
||||
#: these ("London, Ontario" is Canada); a country name does not lose to it.
|
||||
_CITIES: dict[str, str] = {
|
||||
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
|
||||
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
|
||||
**dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck"), "AT"),
|
||||
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege",
|
||||
"luik"), "BE"),
|
||||
**dict.fromkeys(("sofia",), "BG"),
|
||||
**dict.fromkeys(("zagreb",), "HR"),
|
||||
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
|
||||
@@ -369,7 +373,7 @@ _CITIES: dict[str, str] = {
|
||||
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
|
||||
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
|
||||
**dict.fromkeys(("tallinn", "tartu"), "EE"),
|
||||
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
|
||||
**dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"),
|
||||
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
|
||||
"lille", "rennes", "amiens"), "FR"),
|
||||
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
|
||||
@@ -378,27 +382,30 @@ _CITIES: dict[str, str] = {
|
||||
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
|
||||
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
|
||||
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
|
||||
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
|
||||
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck"), "DE"),
|
||||
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
|
||||
**dict.fromkeys(("budapest",), "HU"),
|
||||
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
|
||||
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
|
||||
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
|
||||
"florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples",
|
||||
"napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"),
|
||||
**dict.fromkeys(("riga",), "LV"),
|
||||
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
|
||||
**dict.fromkeys(("luxembourg", "luxemburg"), "LU"),
|
||||
**dict.fromkeys(("valletta",), "MT"),
|
||||
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
||||
"delft"), "NL"),
|
||||
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
|
||||
"PL"),
|
||||
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
|
||||
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
|
||||
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"),
|
||||
**dict.fromkeys(("belgrade", "beograd"), "RS"),
|
||||
**dict.fromkeys(("bratislava",), "SK"),
|
||||
**dict.fromkeys(("ljubljana",), "SI"),
|
||||
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
|
||||
"granada", "santiago de compostela"), "ES"),
|
||||
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
|
||||
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras",
|
||||
"linkoping", "orebro", "norrkoping", "jonkoping"), "SE"),
|
||||
**dict.fromkeys(("reykjavik",), "IS"),
|
||||
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
|
||||
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
|
||||
@@ -408,7 +415,7 @@ _CITIES: dict[str, str] = {
|
||||
"slough", "southampton", "swindon", "liverpool", "redhill",
|
||||
"milton keynes"), "GB"),
|
||||
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
|
||||
"sankt gallen", "st gallen"), "CH"),
|
||||
"sankt gallen", "st gallen", "lucerne", "luzern"), "CH"),
|
||||
**dict.fromkeys(("kyiv", "kiev"), "UA"),
|
||||
**dict.fromkeys(("istanbul", "ankara"), "TR"),
|
||||
**dict.fromkeys(("tirana",), "AL"),
|
||||
@@ -659,9 +666,10 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
|
||||
|
||||
# -------------------------------------------------------------- posting dates
|
||||
|
||||
#: Hosts whose date is a creation or first-publication timestamp, safe to
|
||||
#: read as when a role was posted, checked against live boards on
|
||||
#: 2026-09-29:
|
||||
#: Hosts whose date is a creation or first-publication timestamp, for a
|
||||
#: reader that does not declare `READS` yet (see `trusted_posted`); a reader
|
||||
#: that declares it says so with `fields.posted="trusted"`. Checked against
|
||||
#: live boards on 2026-09-29:
|
||||
#:
|
||||
#: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers
|
||||
#: `first_published` over `updated_at`. A live board (adyen) showed a job
|
||||
@@ -708,14 +716,39 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
|
||||
#: HR Manager) state a date nobody checked, and Pinpoint states none.
|
||||
#:
|
||||
#: A host not listed here, or a role whose stored `published_at` fails the
|
||||
#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the
|
||||
#: safe default this migration is replacing everywhere else.
|
||||
TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({
|
||||
#: sanity check in `posted_at`, keeps `first_seen` as its posted date.
|
||||
_UNDECLARED_TRUSTED: frozenset[str] = frozenset({
|
||||
"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
|
||||
"personio", "teamtailor", "jobindex", "adzuna", "platsbanken",
|
||||
"workday", "successfactors", "oracle", "phenom", "bamboohr", "jobylon", "rippling",
|
||||
})
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _declared_posted() -> dict[str, str]:
|
||||
"""`fields.posted` of every reader that declares `READS`, by name. Read
|
||||
once: the readers are loaded when `plugins.readers` is imported. `plugins`
|
||||
is a top-level folder beside `app`, reached as `app/jobs/extract.py`
|
||||
reaches it."""
|
||||
root = str(Path(__file__).resolve().parents[1])
|
||||
if root not in sys.path:
|
||||
sys.path.insert(0, root)
|
||||
from plugins.readers import READERS
|
||||
|
||||
return {name: module.READS.fields.posted for name, module in READERS.items()
|
||||
if getattr(module, "READS", None) is not None}
|
||||
|
||||
|
||||
def trusted_posted(host: str) -> bool:
|
||||
"""Whether `host` states a date that is safe to read as when a role was
|
||||
posted: its reader declares `fields.posted="trusted"`, or, for a reader
|
||||
without `READS` (or a search provider not yet a reader), it is in
|
||||
`_UNDECLARED_TRUSTED`."""
|
||||
declared = _declared_posted()
|
||||
if host in declared:
|
||||
return declared[host] == "trusted"
|
||||
return host in _UNDECLARED_TRUSTED
|
||||
|
||||
|
||||
#: A `published_at` before this is a parsing slip or a placeholder, not a
|
||||
#: real posting date: no board here existed before 2015.
|
||||
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
||||
@@ -732,7 +765,7 @@ def posted_host(raw: dict[str, Any]) -> str:
|
||||
def posted_at(published_at: str, first_seen: str, host: str) -> str:
|
||||
"""The date a role counts as posted: the earlier of `first_seen` (when
|
||||
the app itself first stored it) and `published_at` (what the board
|
||||
claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and
|
||||
claims), but only when `trusted_posted(host)` and
|
||||
`published_at` is neither absurdly old nor later than `first_seen` — a
|
||||
role cannot be posted after the app already saw it, trusted host or not,
|
||||
which also catches a `published_at` the board reports in the future
|
||||
@@ -740,7 +773,7 @@ def posted_at(published_at: str, first_seen: str, host: str) -> str:
|
||||
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
|
||||
is exact and needs no parsing. A role with no trusted date, or with one
|
||||
that fails the check, keeps `first_seen`."""
|
||||
if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen:
|
||||
if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen:
|
||||
return published_at
|
||||
return first_seen
|
||||
|
||||
@@ -775,6 +808,20 @@ class RoleCopy(Model):
|
||||
posted_at: str = ""
|
||||
|
||||
|
||||
class RolePlace(Model):
|
||||
"""One place of a role (`role_places`): the host's `text`, and the place
|
||||
of the gazetteer it resolves to, `place_id` None (and `name` and `kind`
|
||||
empty) while it resolves to none. `url` is the link of a copy folded into
|
||||
the role, '' for the role's own places."""
|
||||
|
||||
position: int = 0
|
||||
text: str = ""
|
||||
place_id: int | None = None
|
||||
name: str = ""
|
||||
kind: str = ""
|
||||
url: str = ""
|
||||
|
||||
|
||||
class Role(Model):
|
||||
"""A row of `roles`: one posting on one board."""
|
||||
|
||||
@@ -797,13 +844,20 @@ class Role(Model):
|
||||
#: could be normalised into `salary_min`/`salary_max`/`currency`
|
||||
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
|
||||
salary_text: str = ""
|
||||
#: The level shown and filtered on: the first filled of `seniority_model`,
|
||||
#: `seniority_stated` and `seniority_title`.
|
||||
seniority: str = ""
|
||||
seniority_model: str = ""
|
||||
seniority_stated: str = ""
|
||||
seniority_title: str = ""
|
||||
function: str = ""
|
||||
permit: Permit = ""
|
||||
#: ISO 639-1 codes the posting says are required, lowercase.
|
||||
languages: JsonStrings = []
|
||||
description: str = ""
|
||||
feed_text: str = ""
|
||||
#: The short text the host's list carries, until `description` arrives.
|
||||
teaser: str = ""
|
||||
original_url: str = ""
|
||||
published_at: str
|
||||
#: The date this role counts as posted: see `posted_at`. Used for
|
||||
@@ -838,6 +892,9 @@ class Role(Model):
|
||||
#: The open copies of this role, oldest first, when it is the kept one;
|
||||
#: empty otherwise. Not a column: filled by `Store.role`.
|
||||
copies: list[RoleCopy] = []
|
||||
#: Every place of the role, in the host's order. Not a column: filled by
|
||||
#: `Store.role` from `role_places`.
|
||||
places: list[RolePlace] = []
|
||||
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
|
||||
Reference in new issue
Block a user