Filter the feed by company keys, place tokens and seniority, and suggest companies and places
Migration 17 adds roles.place_fold and fills seniority from the title where extract has not read the role. Every connection registers place_match, so the Place filter and the place gate match whole folded words: Lund no longer finds Billund. GET /api/roles takes company_keys, places, seniority, seniority_unknown and pay_unknown; new routes suggest companies and places, label place tokens and count the seniority and pay facets. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
4b9403a0b9
commit
a91b4e99e0
6 files changed
+724
-60
No files matched your search
@@ -17,7 +17,9 @@ import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import country_of, posted_at, posted_host
|
||||
from app.seniority import from_title
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,12 +44,16 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- require the country: "UK | Remote" is not remote from Berlin.
|
||||
place TEXT NOT NULL DEFAULT '',
|
||||
place_key TEXT NOT NULL DEFAULT '',
|
||||
-- places.fold(place): what the Place filter and the place gate match
|
||||
-- whole words against, through the place_match function (app/places.py).
|
||||
place_fold TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
-- seniority.from_title(title) at ingest, until extract reads the posting.
|
||||
seniority TEXT NOT NULL DEFAULT '',
|
||||
function TEXT NOT NULL DEFAULT '',
|
||||
-- Whether the posting says it sponsors a work permit, and the languages
|
||||
@@ -446,6 +452,9 @@ ALTER TABLE companies ADD COLUMN careers_opened_at TEXT NOT NULL DEFAULT '';
|
||||
(16, "companies.hand_reason: why a company is read by hand", """
|
||||
ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
# A Python step: the fold and the title classifier are Python, as in 11.
|
||||
(17, "roles.place_fold for the whole-word place filter, and roles.seniority from "
|
||||
"the title where extract has not read the role", lambda conn: _fold_places(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -523,6 +532,40 @@ def _recompute_country(conn: sqlite3.Connection) -> None:
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def _fold_places(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 17's step. Adds `roles.place_fold` and fills it with
|
||||
`places.fold(place)` for every role, as `Store.upsert_role` writes it for
|
||||
a new one, and fills an empty `seniority` from `seniority.from_title` on
|
||||
every role `extract` has not read (it overwrites the title's level when it
|
||||
does, see `Store.set_extracted`).
|
||||
|
||||
Each distinct place and title is folded or classified once: 34,538 roles
|
||||
hold about 5,600 places. The rows are written with one `executemany` per
|
||||
column inside `_apply`'s transaction, and only those with something to
|
||||
write."""
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN place_fold TEXT NOT NULL DEFAULT ''")
|
||||
rows = conn.execute(
|
||||
"SELECT uid, place, title, seniority, extracted_at FROM roles").fetchall()
|
||||
folds: dict[str, str] = {}
|
||||
levels: dict[str, str] = {}
|
||||
place_updates, level_updates = [], []
|
||||
for uid, place, title, level, extracted_at in rows:
|
||||
if place:
|
||||
if place not in folds:
|
||||
folds[place] = places.fold(place)
|
||||
if folds[place]:
|
||||
place_updates.append((folds[place], uid))
|
||||
if not level and extracted_at is None and title:
|
||||
if title not in levels:
|
||||
levels[title] = from_title(title)
|
||||
if levels[title]:
|
||||
level_updates.append((levels[title], uid))
|
||||
conn.executemany("UPDATE roles SET place_fold = ? WHERE uid = ?", place_updates)
|
||||
conn.executemany("UPDATE roles SET seniority = ? WHERE uid = ?", level_updates)
|
||||
log.info("migration 17: folded %d places over %d roles, a level from the title for %d",
|
||||
len(folds), len(place_updates), len(level_updates))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
Reference in new issue
Block a user