Filter the feed by company keys, place tokens and seniority, and suggest companies and places

Migration 17 adds roles.place_fold and fills seniority from the title where extract has not read the role. Every connection registers place_match, so the Place filter and the place gate match whole folded words: Lund no longer finds Billund. GET /api/roles takes company_keys, places, seniority, seniority_unknown and pay_unknown; new routes suggest companies and places, label place tokens and count the seniority and pay facets.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
WolfandClaude committed 2026-09-29 22:17:12 +02:00
1 parent 4b9403a0b9
commit a91b4e99e0
6 files changed
+724 -60

No files matched your search

+43
View File
@@ -17,7 +17,9 @@ import logging
import sqlite3
from collections.abc import Callable
from app import places
from app.models import country_of, posted_at, posted_host
from app.seniority import from_title
log = logging.getLogger(__name__)
@@ -42,12 +44,16 @@ CREATE TABLE IF NOT EXISTS roles (
-- require the country: "UK | Remote" is not remote from Berlin.
place TEXT NOT NULL DEFAULT '',
place_key TEXT NOT NULL DEFAULT '',
-- places.fold(place): what the Place filter and the place gate match
-- whole words against, through the place_match function (app/places.py).
place_fold TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
-- seniority.from_title(title) at ingest, until extract reads the posting.
seniority TEXT NOT NULL DEFAULT '',
function TEXT NOT NULL DEFAULT '',
-- Whether the posting says it sponsors a work permit, and the languages
@@ -446,6 +452,9 @@ ALTER TABLE companies ADD COLUMN careers_opened_at TEXT NOT NULL DEFAULT '';
(16, "companies.hand_reason: why a company is read by hand", """
ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
"""),
# A Python step: the fold and the title classifier are Python, as in 11.
(17, "roles.place_fold for the whole-word place filter, and roles.seniority from "
"the title where extract has not read the role", lambda conn: _fold_places(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -523,6 +532,40 @@ def _recompute_country(conn: sqlite3.Connection) -> None:
len(updates), len(rows))
def _fold_places(conn: sqlite3.Connection) -> None:
"""Migration 17's step. Adds `roles.place_fold` and fills it with
`places.fold(place)` for every role, as `Store.upsert_role` writes it for
a new one, and fills an empty `seniority` from `seniority.from_title` on
every role `extract` has not read (it overwrites the title's level when it
does, see `Store.set_extracted`).
Each distinct place and title is folded or classified once: 34,538 roles
hold about 5,600 places. The rows are written with one `executemany` per
column inside `_apply`'s transaction, and only those with something to
write."""
conn.execute("ALTER TABLE roles ADD COLUMN place_fold TEXT NOT NULL DEFAULT ''")
rows = conn.execute(
"SELECT uid, place, title, seniority, extracted_at FROM roles").fetchall()
folds: dict[str, str] = {}
levels: dict[str, str] = {}
place_updates, level_updates = [], []
for uid, place, title, level, extracted_at in rows:
if place:
if place not in folds:
folds[place] = places.fold(place)
if folds[place]:
place_updates.append((folds[place], uid))
if not level and extracted_at is None and title:
if title not in levels:
levels[title] = from_title(title)
if levels[title]:
level_updates.append((levels[title], uid))
conn.executemany("UPDATE roles SET place_fold = ? WHERE uid = ?", place_updates)
conn.executemany("UPDATE roles SET seniority = ? WHERE uid = ?", level_updates)
log.info("migration 17: folded %d places over %d roles, a level from the title for %d",
len(folds), len(place_updates), len(level_updates))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
+188 -23
View File
@@ -20,26 +20,30 @@ from uuid import uuid4
import numpy as np
from desk.models import Item
from app import places
from app.models import (
AFFINITY, HAND_REASONS, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, Subscription, Weights,
check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host,
remote_of, title_key,
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole,
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
)
from app.seniority import from_title
from app.storage import schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
#: says less than `extract` found keeps what `extract` found. `function` is
#: not here: it is not a board field (see the note by `keep_filled` below).
_RAW_TEXT = ("place", "country", "currency", "seniority")
#: Nor is `seniority`, which is the title's level unless a board states one.
_RAW_TEXT = ("place", "country", "currency")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
#: structured field wins over what a model read out of the prose, and the next
#: upsert from that board would put it back anyway. `function` is not here
#: either, for the opposite reason — see `set_extracted`.
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "permit")
#: upsert from that board would put it back anyway. `function` and
#: `seniority` are not here either, for the opposite reason — see
#: `set_extracted`.
_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
#: How many replies may leave a role unanswered before `extract` stops
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
@@ -72,6 +76,15 @@ PAY: dict[str, str] = {
"stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)",
}
#: A role that states no pay at all, what `pay_unknown` lets through the Pay
#: filter, and the `""` of its facet.
_NO_PAY = "(salary_min IS NULL AND salary_max IS NULL AND salary_text = '')"
#: `salary_state` in SQL, what the Pay facet counts by: 'normalised',
#: 'stated' (a line and no figure) or ''.
_PAY_STATE = ("CASE WHEN salary_min IS NOT NULL OR salary_max IS NOT NULL THEN 'normalised' "
"WHEN salary_text != '' THEN 'stated' ELSE '' END")
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
#: moves the role at once and the stored score stays what `score` computed.
@@ -173,8 +186,12 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "place":
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
return "r.place = ''", f"({likes})", [_like(v) for v in values]
# Whole folded words, as the feed's Place filter: `lund` is not in
# `Billund`. A value is a place token, bare text a `place:`.
clauses = [places.match_sql(v, place="r.place_fold", country="r.country")
for v in values]
match = " OR ".join(sql for sql, _ in clauses) or "0"
return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps]
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
@@ -186,6 +203,20 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]:
"""The Place filter over `roles`: any of the tokens, each a `place_match`
call (`places.match_sql`). ValueError for a token that names nothing,
which would otherwise match no role without saying why."""
clauses, params = [], []
for token in tokens:
if not places.known(token):
raise ValueError(f"{token!r} is not a place this app knows")
sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country")
clauses.append(sql)
params += values
return f"({' OR '.join(clauses)})", params
def _int_or_none(value: Any) -> int | None:
if value is None or value == "" or isinstance(value, bool):
return None
@@ -242,9 +273,12 @@ class Store:
def _connect(self) -> sqlite3.Connection:
"""A fresh connection. `foreign_keys` is off by default per connection;
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion."""
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
registered on every one, since the place filter and the place gate
call it (`places.match_sql`)."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
places.register(conn)
conn.execute("PRAGMA foreign_keys=ON")
conn.execute("PRAGMA busy_timeout=30000")
conn.execute("PRAGMA synchronous=NORMAL")
@@ -302,6 +336,14 @@ class Store:
`extract` (`extracted_at` is set), this upsert leaves `function`
alone for good, whatever the plugin's raw guess now says — see
`set_extracted` for the other half of the rule.
`seniority` is the title's level (`seniority.from_title`) until
extract has read the role and extract's from then on, as `function`
is. The classifier is called here rather than in a plugin, so a role
from any source gets it, the way `country` comes from `place` here
too. A level the board states in `raw` is a board field and wins over
both. `place_fold` is written with `place`, under the same keep-filled
rule.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
@@ -336,6 +378,9 @@ class Store:
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
stated = str(raw.get("seniority") or "")
row["seniority"] = stated or from_title(item.title)
row["place_fold"] = places.fold(row["place"])
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
# must be able to clear it, not leave a stale line behind.
@@ -359,6 +404,13 @@ class Store:
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
"ELSE roles.function END")
# The title's level follows the title until extract has read the
# role, and is extract's from then on; a board's stated level wins.
updates.append("seniority=excluded.seniority" if stated else
"seniority=CASE WHEN roles.extracted_at IS NULL "
"THEN excluded.seniority ELSE roles.seniority END")
updates.append("place_fold=CASE WHEN excluded.place != '' "
"THEN excluded.place_fold ELSE roles.place_fold END")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
@@ -492,7 +544,9 @@ class Store:
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, pay: str | None = None,
offset: int = 0) -> list[ScoredRole]:
company_keys: Sequence[str] = (), places: Sequence[str] = (),
seniority: Sequence[str] = (), seniority_unknown: bool = True,
pay_unknown: bool = False, offset: int = 0) -> list[ScoredRole]:
"""Newest first by `posted_at`. `company` and `place` are
case-insensitive substrings of the columns a board wrote — not the
key, so a search does not have to know the exact slug a company folds
@@ -500,7 +554,15 @@ class Store:
role must match every word in one or the other. `function` and
`remote` match a gate column exactly. `days` keeps only roles posted
within that many days of now. `pay` (a key of `PAY`) filters on
`salary_state`.
`salary_state`; with `pay_unknown` a role that states no pay passes
it too.
`company_keys` keeps those exact companies. `places` keeps a role any
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
`place:malmo`), matched on whole folded words; a token that names
nothing is a ValueError. `seniority` keeps the roles at any of those
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
with no level, which is most of them. An empty list is no filter.
`agent` says whose votes, hidden companies and scores are meant. Each
row carries that agent's `vote` and `voted_at`; `judged` (a key of
@@ -535,7 +597,8 @@ class Store:
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged, pay=pay)
judged=judged, pay=pay, company_keys=company_keys, places=places,
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown)
select = "SELECT roles.*, v.vote, v.voted_at"
order = "roles.posted_at DESC, roles.uid"
scored = agent is not None and by_score
@@ -566,7 +629,9 @@ class Store:
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, pay: str | None = None,
offset: int = 0) -> list[RoleRow]:
company_keys: Sequence[str] = (), places: Sequence[str] = (),
seniority: Sequence[str] = (), seniority_unknown: bool = True,
pay_unknown: bool = False, offset: int = 0) -> list[RoleRow]:
"""What `GET /api/roles` serves: the same rows `roles` would, in the
same order, with the same filters and paging, but only the fields a
row draws (see `RoleRow`) — never `description`, `feed_text`, `raw`
@@ -581,10 +646,11 @@ class Store:
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged, pay=pay)
judged=judged, pay=pay, company_keys=company_keys, places=places,
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown)
select = (
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
"roles.country, roles.remote, roles.function, roles.salary_min, "
"roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, "
"roles.salary_max, roles.currency, roles.salary_text, roles.original_url, "
"roles.posted_at, roles.read_at, "
"v.vote, v.voted_at, "
@@ -617,22 +683,64 @@ class Store:
include_read: bool = True, tags: Sequence[str] = (),
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
gated: bool = False, judged: str | None = None,
pay: str | None = None) -> int:
pay: str | None = None, company_keys: Sequence[str] = (),
places: Sequence[str] = (), seniority: Sequence[str] = (),
seniority_unknown: bool = True, pay_unknown: bool = False) -> int:
"""How many roles `roles` would list with the same filters and no limit."""
sql, params = self._roles_from(
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged, pay=pay)
judged=judged, pay=pay, company_keys=company_keys, places=places,
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown)
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
return int(row["n"]) if row else 0
def role_facets(self, company: str | None = None, place: str | None = None,
open_only: bool = True, q: str | None = None,
function: str | None = None, remote: str | None = None,
days: int | None = None, hide_muted: bool = False,
include_read: bool = True, tags: Sequence[str] = (),
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
gated: bool = False, judged: str | None = None,
pay: str | None = None, company_keys: Sequence[str] = (),
places: Sequence[str] = (), seniority: Sequence[str] = (),
seniority_unknown: bool = True,
pay_unknown: bool = False) -> dict[str, list[tuple[str, int]]]:
"""How many roles `roles` would list at each seniority level and each
pay state, each counted with every other filter and its own left
out, so the rail can say what a click would add. `seniority` is every
level of `LEVELS` in order and then `""`, no level; `pay` is
`salary_state`'s 'normalised' and 'stated' (a line with no figure)
and then `""`, no pay stated. A value no role has counts 0."""
shared: dict[str, Any] = {
"company": company, "place": place, "open_only": open_only, "q": q,
"function": function, "remote": remote, "days": days, "hide_muted": hide_muted,
"include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent,
"by_score": by_score, "gated": gated, "judged": judged,
"company_keys": company_keys, "places": places}
by_level, level_params = self._roles_from(
**shared, pay=pay, pay_unknown=pay_unknown)
by_pay, pay_params = self._roles_from(
**shared, seniority=seniority, seniority_unknown=seniority_unknown)
levels = {r["value"]: int(r["n"]) for r in self._rows(
f"SELECT roles.seniority AS value, COUNT(*) AS n {by_level} "
"GROUP BY roles.seniority", level_params)}
pays = {r["value"]: int(r["n"]) for r in self._rows(
f"SELECT {_PAY_STATE} AS value, COUNT(*) AS n {by_pay} GROUP BY value",
pay_params)}
return {"seniority": [(v, levels.get(v, 0)) for v in (*LEVELS, "")],
"pay": [(v, pays.get(v, 0)) for v in ("normalised", "stated", "")]}
def _roles_from(self, *, company: str | None, place: str | None, open_only: bool,
q: str | None, function: str | None, remote: str | None,
days: int | None, hide_muted: bool, include_read: bool,
tags: Sequence[str], tag_mode: str, agent: int | None,
by_score: bool, gated: bool, judged: str | None,
pay: str | None = None) -> tuple[str, list[Any]]:
pay: str | None = None, company_keys: Sequence[str] = (),
places: Sequence[str] = (), seniority: Sequence[str] = (),
seniority_unknown: bool = True,
pay_unknown: bool = False) -> tuple[str, list[Any]]:
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
its parameters in order. The agent's votes are always joined as `v`
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
@@ -663,7 +771,16 @@ class Store:
if pay:
if pay not in PAY:
raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}")
where.append(PAY[pay])
where.append(f"({PAY[pay]} OR {_NO_PAY})" if pay_unknown else PAY[pay])
if company_keys:
where.append(f"company_key IN ({', '.join('?' for _ in company_keys)})")
params += list(company_keys)
if seniority:
if unknown := sorted(set(seniority) - set(LEVELS)):
raise ValueError(f"seniority is {', '.join(LEVELS)}, not {unknown[0]!r}")
levels = f"roles.seniority IN ({', '.join('?' for _ in seniority)})"
where.append(f"({levels} OR roles.seniority = '')" if seniority_unknown else levels)
params += list(seniority)
if open_only:
where.append("closed_at IS NULL")
if hide_muted and agent is not None:
@@ -686,6 +803,11 @@ class Store:
params += list(tags)
else:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
if places:
# Last, so the cheaper terms have cut the rows it is called on.
clause, values = _places_sql(places)
where.append(clause)
params += values
sql = f"FROM roles {_WITH_VOTES}"
joins: list[Any] = [agent]
if agent is not None and by_score:
@@ -824,7 +946,7 @@ class Store:
Only into empty columns, and only a non-empty value: a board's own
field is never replaced and nothing is ever blanked. `place` brings
`place_key` with it. Returns how many columns were filled, or None
`place_key` and `place_fold` with it. Returns how many columns were filled, or None
for an unknown uid. Values are trusted to be in vocabulary; the
job's `apply` is where a reply is checked.
@@ -832,7 +954,10 @@ class Store:
the `ats` classifier's keyword guess, not a board field, so a
non-empty answer overwrites it outright rather than only filling an
empty column — see `upsert_role`'s docstring for the other half of
the rule that then keeps a later pull from putting the guess back."""
the rule that then keeps a later pull from putting the guess back.
`seniority` is the other: `upsert_role` fills it from the title, so a
non-empty answer replaces it, unless the board stated a level in
`raw`."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -845,6 +970,9 @@ class Store:
function = str(fields.get("function") or "")
if function:
sets["function"] = function
level = str(fields.get("seniority") or "")
if level and not json.loads(row["raw"] or "{}").get("seniority"):
sets["seniority"] = level
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
@@ -855,6 +983,7 @@ class Store:
filled = len(sets)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
@@ -1273,6 +1402,42 @@ class Store:
"SELECT company_key, COUNT(*) AS n FROM roles "
"WHERE closed_at IS NULL GROUP BY company_key")}
def open_companies(self, agent_id: int | None = None
) -> list[tuple[str, list[str], int, int]]:
"""`(company_key, names, open roles, affinity)` for every company with
an open role, what the Company field suggests from. `names` are the
company row's name and then every spelling its roles carry, commonest
first, so a search finds a company by any of them. A company
`agent_id` hid is left out; `affinity` is that agent's, 0 without
one."""
rows = self._rows(
"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n,
c.name AS own, COALESCE(ac.affinity, 0) AS affinity
FROM roles r
LEFT JOIN companies c ON c.key = r.company_key
LEFT JOIN agent_companies ac ON ac.company_key = r.company_key
AND ac.agent_id = ?
WHERE r.closed_at IS NULL AND r.company_key != ''
AND COALESCE(ac.affinity, 0) != -1
GROUP BY r.company_key, r.company
ORDER BY r.company_key, n DESC, r.company""", (agent_id,))
found: dict[str, tuple[list[str], list[int], int]] = {}
for r in rows:
names, counts, _ = found.setdefault(
r["key"], ([r["own"]] if r["own"] else [], [], int(r["affinity"])))
if r["name"] and r["name"] not in names:
names.append(r["name"])
counts.append(int(r["n"]))
return [(key, names, sum(counts), affinity)
for key, (names, counts, affinity) in found.items()]
def open_places(self) -> list[tuple[str, str, int]]:
"""`(place, country, open roles)` for every place line an open role
carries, what `places.suggest` reads."""
return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows(
"SELECT place, country, COUNT(*) AS n FROM roles "
"WHERE closed_at IS NULL AND place != '' GROUP BY place, country")]
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
"""Per subscription id, its open roles counted by `country`."""
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles