Filter the feed by company keys, place tokens and seniority, and suggest companies and places

Migration 17 adds roles.place_fold and fills seniority from the title where extract has not read the role. Every connection registers place_match, so the Place filter and the place gate match whole folded words: Lund no longer finds Billund. GET /api/roles takes company_keys, places, seniority, seniority_unknown and pay_unknown; new routes suggest companies and places, label place tokens and count the seniority and pay facets.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
WolfandClaude committed 2026-09-29 22:17:12 +02:00
1 parent 4b9403a0b9
commit a91b4e99e0
6 files changed
+724 -60

No files matched your search

+43
View File
@@ -17,7 +17,9 @@ import logging
import sqlite3
from collections.abc import Callable
from app import places
from app.models import country_of, posted_at, posted_host
from app.seniority import from_title
log = logging.getLogger(__name__)
@@ -42,12 +44,16 @@ CREATE TABLE IF NOT EXISTS roles (
-- require the country: "UK | Remote" is not remote from Berlin.
place TEXT NOT NULL DEFAULT '',
place_key TEXT NOT NULL DEFAULT '',
-- places.fold(place): what the Place filter and the place gate match
-- whole words against, through the place_match function (app/places.py).
place_fold TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
-- seniority.from_title(title) at ingest, until extract reads the posting.
seniority TEXT NOT NULL DEFAULT '',
function TEXT NOT NULL DEFAULT '',
-- Whether the posting says it sponsors a work permit, and the languages
@@ -446,6 +452,9 @@ ALTER TABLE companies ADD COLUMN careers_opened_at TEXT NOT NULL DEFAULT '';
(16, "companies.hand_reason: why a company is read by hand", """
ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
"""),
# A Python step: the fold and the title classifier are Python, as in 11.
(17, "roles.place_fold for the whole-word place filter, and roles.seniority from "
"the title where extract has not read the role", lambda conn: _fold_places(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -523,6 +532,40 @@ def _recompute_country(conn: sqlite3.Connection) -> None:
len(updates), len(rows))
def _fold_places(conn: sqlite3.Connection) -> None:
"""Migration 17's step. Adds `roles.place_fold` and fills it with
`places.fold(place)` for every role, as `Store.upsert_role` writes it for
a new one, and fills an empty `seniority` from `seniority.from_title` on
every role `extract` has not read (it overwrites the title's level when it
does, see `Store.set_extracted`).
Each distinct place and title is folded or classified once: 34,538 roles
hold about 5,600 places. The rows are written with one `executemany` per
column inside `_apply`'s transaction, and only those with something to
write."""
conn.execute("ALTER TABLE roles ADD COLUMN place_fold TEXT NOT NULL DEFAULT ''")
rows = conn.execute(
"SELECT uid, place, title, seniority, extracted_at FROM roles").fetchall()
folds: dict[str, str] = {}
levels: dict[str, str] = {}
place_updates, level_updates = [], []
for uid, place, title, level, extracted_at in rows:
if place:
if place not in folds:
folds[place] = places.fold(place)
if folds[place]:
place_updates.append((folds[place], uid))
if not level and extracted_at is None and title:
if title not in levels:
levels[title] = from_title(title)
if levels[title]:
level_updates.append((levels[title], uid))
conn.executemany("UPDATE roles SET place_fold = ? WHERE uid = ?", place_updates)
conn.executemany("UPDATE roles SET seniority = ? WHERE uid = ?", level_updates)
log.info("migration 17: folded %d places over %d roles, a level from the title for %d",
len(folds), len(place_updates), len(level_updates))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])