Merge branch 'wp-e-api'
This commit is contained in:
commit
fd89381caa
6 files changed
+724
-60
No files matched your search
+8
-2
@@ -632,6 +632,8 @@ def remote_of(place: str) -> str:
|
||||
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
|
||||
"director", "exec"]
|
||||
SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
||||
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
|
||||
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
|
||||
|
||||
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
||||
Permit = Literal["", "sponsors", "no_sponsorship"]
|
||||
@@ -891,7 +893,9 @@ class RoleRow(Model):
|
||||
characters. The pay fields are what the row's badge reads: the three
|
||||
normalised columns, the line the board stated, and `salary_state`,
|
||||
derived as on `Role`. `original_url` is one short string, kept so the
|
||||
`o` key opens a selected row's posting without opening the reader."""
|
||||
`o` key opens a selected row's posting without opening the reader.
|
||||
`seniority` is the title's level until extract reads the posting, ''
|
||||
when neither says."""
|
||||
|
||||
uid: str
|
||||
company_key: str = ""
|
||||
@@ -901,6 +905,7 @@ class RoleRow(Model):
|
||||
country: str = ""
|
||||
remote: Remote = ""
|
||||
function: str = ""
|
||||
seniority: str = ""
|
||||
salary_min: int | None = None
|
||||
salary_max: int | None = None
|
||||
currency: str = ""
|
||||
@@ -998,7 +1003,8 @@ class Application(Model):
|
||||
# ------------------------------------------------------------------ agents
|
||||
|
||||
#: The role columns a gate can name. `company` matches on the company key;
|
||||
#: `place` is a case-insensitive substring, like the place filter; the rest are
|
||||
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
|
||||
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are
|
||||
#: case-insensitive equality, except `salary_min` (at least the value) and
|
||||
#: `languages` (every language the role requires is one of the values).
|
||||
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
|
||||
|
||||
@@ -17,7 +17,9 @@ import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import country_of, posted_at, posted_host
|
||||
from app.seniority import from_title
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,12 +44,16 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- require the country: "UK | Remote" is not remote from Berlin.
|
||||
place TEXT NOT NULL DEFAULT '',
|
||||
place_key TEXT NOT NULL DEFAULT '',
|
||||
-- places.fold(place): what the Place filter and the place gate match
|
||||
-- whole words against, through the place_match function (app/places.py).
|
||||
place_fold TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
-- seniority.from_title(title) at ingest, until extract reads the posting.
|
||||
seniority TEXT NOT NULL DEFAULT '',
|
||||
function TEXT NOT NULL DEFAULT '',
|
||||
-- Whether the posting says it sponsors a work permit, and the languages
|
||||
@@ -446,6 +452,9 @@ ALTER TABLE companies ADD COLUMN careers_opened_at TEXT NOT NULL DEFAULT '';
|
||||
(16, "companies.hand_reason: why a company is read by hand", """
|
||||
ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
# A Python step: the fold and the title classifier are Python, as in 11.
|
||||
(17, "roles.place_fold for the whole-word place filter, and roles.seniority from "
|
||||
"the title where extract has not read the role", lambda conn: _fold_places(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -523,6 +532,40 @@ def _recompute_country(conn: sqlite3.Connection) -> None:
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def _fold_places(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 17's step. Adds `roles.place_fold` and fills it with
|
||||
`places.fold(place)` for every role, as `Store.upsert_role` writes it for
|
||||
a new one, and fills an empty `seniority` from `seniority.from_title` on
|
||||
every role `extract` has not read (it overwrites the title's level when it
|
||||
does, see `Store.set_extracted`).
|
||||
|
||||
Each distinct place and title is folded or classified once: 34,538 roles
|
||||
hold about 5,600 places. The rows are written with one `executemany` per
|
||||
column inside `_apply`'s transaction, and only those with something to
|
||||
write."""
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN place_fold TEXT NOT NULL DEFAULT ''")
|
||||
rows = conn.execute(
|
||||
"SELECT uid, place, title, seniority, extracted_at FROM roles").fetchall()
|
||||
folds: dict[str, str] = {}
|
||||
levels: dict[str, str] = {}
|
||||
place_updates, level_updates = [], []
|
||||
for uid, place, title, level, extracted_at in rows:
|
||||
if place:
|
||||
if place not in folds:
|
||||
folds[place] = places.fold(place)
|
||||
if folds[place]:
|
||||
place_updates.append((folds[place], uid))
|
||||
if not level and extracted_at is None and title:
|
||||
if title not in levels:
|
||||
levels[title] = from_title(title)
|
||||
if levels[title]:
|
||||
level_updates.append((levels[title], uid))
|
||||
conn.executemany("UPDATE roles SET place_fold = ? WHERE uid = ?", place_updates)
|
||||
conn.executemany("UPDATE roles SET seniority = ? WHERE uid = ?", level_updates)
|
||||
log.info("migration 17: folded %d places over %d roles, a level from the title for %d",
|
||||
len(folds), len(place_updates), len(level_updates))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+188
-23
@@ -20,26 +20,30 @@ from uuid import uuid4
|
||||
import numpy as np
|
||||
|
||||
from desk.models import Item
|
||||
from app import places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
||||
Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, Subscription, Weights,
|
||||
check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host,
|
||||
remote_of, title_key,
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
#: empty value never overwrites a filled column, so an upsert from a board that
|
||||
#: says less than `extract` found keeps what `extract` found. `function` is
|
||||
#: not here: it is not a board field (see the note by `keep_filled` below).
|
||||
_RAW_TEXT = ("place", "country", "currency", "seniority")
|
||||
#: Nor is `seniority`, which is the title's level unless a board states one.
|
||||
_RAW_TEXT = ("place", "country", "currency")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
#: structured field wins over what a model read out of the prose, and the next
|
||||
#: upsert from that board would put it back anyway. `function` is not here
|
||||
#: either, for the opposite reason — see `set_extracted`.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "permit")
|
||||
#: upsert from that board would put it back anyway. `function` and
|
||||
#: `seniority` are not here either, for the opposite reason — see
|
||||
#: `set_extracted`.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
|
||||
|
||||
#: How many replies may leave a role unanswered before `extract` stops
|
||||
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
||||
@@ -72,6 +76,15 @@ PAY: dict[str, str] = {
|
||||
"stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)",
|
||||
}
|
||||
|
||||
#: A role that states no pay at all, what `pay_unknown` lets through the Pay
|
||||
#: filter, and the `""` of its facet.
|
||||
_NO_PAY = "(salary_min IS NULL AND salary_max IS NULL AND salary_text = '')"
|
||||
|
||||
#: `salary_state` in SQL, what the Pay facet counts by: 'normalised',
|
||||
#: 'stated' (a line and no figure) or ''.
|
||||
_PAY_STATE = ("CASE WHEN salary_min IS NOT NULL OR salary_max IS NOT NULL THEN 'normalised' "
|
||||
"WHEN salary_text != '' THEN 'stated' ELSE '' END")
|
||||
|
||||
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
|
||||
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
|
||||
#: moves the role at once and the stored score stays what `score` computed.
|
||||
@@ -173,8 +186,12 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
||||
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
||||
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
||||
if gate.field == "place":
|
||||
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
|
||||
return "r.place = ''", f"({likes})", [_like(v) for v in values]
|
||||
# Whole folded words, as the feed's Place filter: `lund` is not in
|
||||
# `Billund`. A value is a place token, bare text a `place:`.
|
||||
clauses = [places.match_sql(v, place="r.place_fold", country="r.country")
|
||||
for v in values]
|
||||
match = " OR ".join(sql for sql, _ in clauses) or "0"
|
||||
return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps]
|
||||
if gate.field == "company":
|
||||
keys = [company_key(v) for v in values]
|
||||
return ("r.company_key = ''",
|
||||
@@ -186,6 +203,20 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||
|
||||
|
||||
def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]:
|
||||
"""The Place filter over `roles`: any of the tokens, each a `place_match`
|
||||
call (`places.match_sql`). ValueError for a token that names nothing,
|
||||
which would otherwise match no role without saying why."""
|
||||
clauses, params = [], []
|
||||
for token in tokens:
|
||||
if not places.known(token):
|
||||
raise ValueError(f"{token!r} is not a place this app knows")
|
||||
sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country")
|
||||
clauses.append(sql)
|
||||
params += values
|
||||
return f"({' OR '.join(clauses)})", params
|
||||
|
||||
|
||||
def _int_or_none(value: Any) -> int | None:
|
||||
if value is None or value == "" or isinstance(value, bool):
|
||||
return None
|
||||
@@ -242,9 +273,12 @@ class Store:
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
"""A fresh connection. `foreign_keys` is off by default per connection;
|
||||
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
||||
`synchronous=NORMAL` is the standard WAL companion."""
|
||||
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
|
||||
registered on every one, since the place filter and the place gate
|
||||
call it (`places.match_sql`)."""
|
||||
conn = sqlite3.connect(self.path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
places.register(conn)
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute("PRAGMA busy_timeout=30000")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
@@ -302,6 +336,14 @@ class Store:
|
||||
`extract` (`extracted_at` is set), this upsert leaves `function`
|
||||
alone for good, whatever the plugin's raw guess now says — see
|
||||
`set_extracted` for the other half of the rule.
|
||||
|
||||
`seniority` is the title's level (`seniority.from_title`) until
|
||||
extract has read the role and extract's from then on, as `function`
|
||||
is. The classifier is called here rather than in a plugin, so a role
|
||||
from any source gets it, the way `country` comes from `place` here
|
||||
too. A level the board states in `raw` is a board field and wins over
|
||||
both. `place_fold` is written with `place`, under the same keep-filled
|
||||
rule.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
@@ -336,6 +378,9 @@ class Store:
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
stated = str(raw.get("seniority") or "")
|
||||
row["seniority"] = stated or from_title(item.title)
|
||||
row["place_fold"] = places.fold(row["place"])
|
||||
# A board's own pay line, one per pull like title or published_at —
|
||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||
# must be able to clear it, not leave a stale line behind.
|
||||
@@ -359,6 +404,13 @@ class Store:
|
||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||
"ELSE roles.function END")
|
||||
# The title's level follows the title until extract has read the
|
||||
# role, and is extract's from then on; a board's stated level wins.
|
||||
updates.append("seniority=excluded.seniority" if stated else
|
||||
"seniority=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN excluded.seniority ELSE roles.seniority END")
|
||||
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
||||
"THEN excluded.place_fold ELSE roles.place_fold END")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
@@ -492,7 +544,9 @@ class Store:
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, pay: str | None = None,
|
||||
offset: int = 0) -> list[ScoredRole]:
|
||||
company_keys: Sequence[str] = (), places: Sequence[str] = (),
|
||||
seniority: Sequence[str] = (), seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False, offset: int = 0) -> list[ScoredRole]:
|
||||
"""Newest first by `posted_at`. `company` and `place` are
|
||||
case-insensitive substrings of the columns a board wrote — not the
|
||||
key, so a search does not have to know the exact slug a company folds
|
||||
@@ -500,7 +554,15 @@ class Store:
|
||||
role must match every word in one or the other. `function` and
|
||||
`remote` match a gate column exactly. `days` keeps only roles posted
|
||||
within that many days of now. `pay` (a key of `PAY`) filters on
|
||||
`salary_state`.
|
||||
`salary_state`; with `pay_unknown` a role that states no pay passes
|
||||
it too.
|
||||
|
||||
`company_keys` keeps those exact companies. `places` keeps a role any
|
||||
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
|
||||
`place:malmo`), matched on whole folded words; a token that names
|
||||
nothing is a ValueError. `seniority` keeps the roles at any of those
|
||||
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
|
||||
with no level, which is most of them. An empty list is no filter.
|
||||
|
||||
`agent` says whose votes, hidden companies and scores are meant. Each
|
||||
row carries that agent's `vote` and `voted_at`; `judged` (a key of
|
||||
@@ -535,7 +597,8 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged, pay=pay)
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown)
|
||||
select = "SELECT roles.*, v.vote, v.voted_at"
|
||||
order = "roles.posted_at DESC, roles.uid"
|
||||
scored = agent is not None and by_score
|
||||
@@ -566,7 +629,9 @@ class Store:
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, pay: str | None = None,
|
||||
offset: int = 0) -> list[RoleRow]:
|
||||
company_keys: Sequence[str] = (), places: Sequence[str] = (),
|
||||
seniority: Sequence[str] = (), seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False, offset: int = 0) -> list[RoleRow]:
|
||||
"""What `GET /api/roles` serves: the same rows `roles` would, in the
|
||||
same order, with the same filters and paging, but only the fields a
|
||||
row draws (see `RoleRow`) — never `description`, `feed_text`, `raw`
|
||||
@@ -581,10 +646,11 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged, pay=pay)
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown)
|
||||
select = (
|
||||
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
|
||||
"roles.country, roles.remote, roles.function, roles.salary_min, "
|
||||
"roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, "
|
||||
"roles.salary_max, roles.currency, roles.salary_text, roles.original_url, "
|
||||
"roles.posted_at, roles.read_at, "
|
||||
"v.vote, v.voted_at, "
|
||||
@@ -617,22 +683,64 @@ class Store:
|
||||
include_read: bool = True, tags: Sequence[str] = (),
|
||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||
gated: bool = False, judged: str | None = None,
|
||||
pay: str | None = None) -> int:
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True, pay_unknown: bool = False) -> int:
|
||||
"""How many roles `roles` would list with the same filters and no limit."""
|
||||
sql, params = self._roles_from(
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged, pay=pay)
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown)
|
||||
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def role_facets(self, company: str | None = None, place: str | None = None,
|
||||
open_only: bool = True, q: str | None = None,
|
||||
function: str | None = None, remote: str | None = None,
|
||||
days: int | None = None, hide_muted: bool = False,
|
||||
include_read: bool = True, tags: Sequence[str] = (),
|
||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||
gated: bool = False, judged: str | None = None,
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False) -> dict[str, list[tuple[str, int]]]:
|
||||
"""How many roles `roles` would list at each seniority level and each
|
||||
pay state, each counted with every other filter and its own left
|
||||
out, so the rail can say what a click would add. `seniority` is every
|
||||
level of `LEVELS` in order and then `""`, no level; `pay` is
|
||||
`salary_state`'s 'normalised' and 'stated' (a line with no figure)
|
||||
and then `""`, no pay stated. A value no role has counts 0."""
|
||||
shared: dict[str, Any] = {
|
||||
"company": company, "place": place, "open_only": open_only, "q": q,
|
||||
"function": function, "remote": remote, "days": days, "hide_muted": hide_muted,
|
||||
"include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent,
|
||||
"by_score": by_score, "gated": gated, "judged": judged,
|
||||
"company_keys": company_keys, "places": places}
|
||||
by_level, level_params = self._roles_from(
|
||||
**shared, pay=pay, pay_unknown=pay_unknown)
|
||||
by_pay, pay_params = self._roles_from(
|
||||
**shared, seniority=seniority, seniority_unknown=seniority_unknown)
|
||||
levels = {r["value"]: int(r["n"]) for r in self._rows(
|
||||
f"SELECT roles.seniority AS value, COUNT(*) AS n {by_level} "
|
||||
"GROUP BY roles.seniority", level_params)}
|
||||
pays = {r["value"]: int(r["n"]) for r in self._rows(
|
||||
f"SELECT {_PAY_STATE} AS value, COUNT(*) AS n {by_pay} GROUP BY value",
|
||||
pay_params)}
|
||||
return {"seniority": [(v, levels.get(v, 0)) for v in (*LEVELS, "")],
|
||||
"pay": [(v, pays.get(v, 0)) for v in ("normalised", "stated", "")]}
|
||||
|
||||
def _roles_from(self, *, company: str | None, place: str | None, open_only: bool,
|
||||
q: str | None, function: str | None, remote: str | None,
|
||||
days: int | None, hide_muted: bool, include_read: bool,
|
||||
tags: Sequence[str], tag_mode: str, agent: int | None,
|
||||
by_score: bool, gated: bool, judged: str | None,
|
||||
pay: str | None = None) -> tuple[str, list[Any]]:
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False) -> tuple[str, list[Any]]:
|
||||
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
||||
its parameters in order. The agent's votes are always joined as `v`
|
||||
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
|
||||
@@ -663,7 +771,16 @@ class Store:
|
||||
if pay:
|
||||
if pay not in PAY:
|
||||
raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}")
|
||||
where.append(PAY[pay])
|
||||
where.append(f"({PAY[pay]} OR {_NO_PAY})" if pay_unknown else PAY[pay])
|
||||
if company_keys:
|
||||
where.append(f"company_key IN ({', '.join('?' for _ in company_keys)})")
|
||||
params += list(company_keys)
|
||||
if seniority:
|
||||
if unknown := sorted(set(seniority) - set(LEVELS)):
|
||||
raise ValueError(f"seniority is {', '.join(LEVELS)}, not {unknown[0]!r}")
|
||||
levels = f"roles.seniority IN ({', '.join('?' for _ in seniority)})"
|
||||
where.append(f"({levels} OR roles.seniority = '')" if seniority_unknown else levels)
|
||||
params += list(seniority)
|
||||
if open_only:
|
||||
where.append("closed_at IS NULL")
|
||||
if hide_muted and agent is not None:
|
||||
@@ -686,6 +803,11 @@ class Store:
|
||||
params += list(tags)
|
||||
else:
|
||||
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
|
||||
if places:
|
||||
# Last, so the cheaper terms have cut the rows it is called on.
|
||||
clause, values = _places_sql(places)
|
||||
where.append(clause)
|
||||
params += values
|
||||
sql = f"FROM roles {_WITH_VOTES}"
|
||||
joins: list[Any] = [agent]
|
||||
if agent is not None and by_score:
|
||||
@@ -824,7 +946,7 @@ class Store:
|
||||
|
||||
Only into empty columns, and only a non-empty value: a board's own
|
||||
field is never replaced and nothing is ever blanked. `place` brings
|
||||
`place_key` with it. Returns how many columns were filled, or None
|
||||
`place_key` and `place_fold` with it. Returns how many columns were filled, or None
|
||||
for an unknown uid. Values are trusted to be in vocabulary; the
|
||||
job's `apply` is where a reply is checked.
|
||||
|
||||
@@ -832,7 +954,10 @@ class Store:
|
||||
the `ats` classifier's keyword guess, not a board field, so a
|
||||
non-empty answer overwrites it outright rather than only filling an
|
||||
empty column — see `upsert_role`'s docstring for the other half of
|
||||
the rule that then keeps a later pull from putting the guess back."""
|
||||
the rule that then keeps a later pull from putting the guess back.
|
||||
`seniority` is the other: `upsert_role` fills it from the title, so a
|
||||
non-empty answer replaces it, unless the board stated a level in
|
||||
`raw`."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
@@ -845,6 +970,9 @@ class Store:
|
||||
function = str(fields.get("function") or "")
|
||||
if function:
|
||||
sets["function"] = function
|
||||
level = str(fields.get("seniority") or "")
|
||||
if level and not json.loads(row["raw"] or "{}").get("seniority"):
|
||||
sets["seniority"] = level
|
||||
for name in _RAW_INT:
|
||||
number = _int_or_none(fields.get(name))
|
||||
if number is not None and row[name] is None:
|
||||
@@ -855,6 +983,7 @@ class Store:
|
||||
filled = len(sets)
|
||||
if "place" in sets:
|
||||
sets["place_key"] = place_key(sets["place"])
|
||||
sets["place_fold"] = places.fold(sets["place"])
|
||||
sets["extracted_at"] = self.now()
|
||||
conn.execute(
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
||||
@@ -1273,6 +1402,42 @@ class Store:
|
||||
"SELECT company_key, COUNT(*) AS n FROM roles "
|
||||
"WHERE closed_at IS NULL GROUP BY company_key")}
|
||||
|
||||
def open_companies(self, agent_id: int | None = None
|
||||
) -> list[tuple[str, list[str], int, int]]:
|
||||
"""`(company_key, names, open roles, affinity)` for every company with
|
||||
an open role, what the Company field suggests from. `names` are the
|
||||
company row's name and then every spelling its roles carry, commonest
|
||||
first, so a search finds a company by any of them. A company
|
||||
`agent_id` hid is left out; `affinity` is that agent's, 0 without
|
||||
one."""
|
||||
rows = self._rows(
|
||||
"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n,
|
||||
c.name AS own, COALESCE(ac.affinity, 0) AS affinity
|
||||
FROM roles r
|
||||
LEFT JOIN companies c ON c.key = r.company_key
|
||||
LEFT JOIN agent_companies ac ON ac.company_key = r.company_key
|
||||
AND ac.agent_id = ?
|
||||
WHERE r.closed_at IS NULL AND r.company_key != ''
|
||||
AND COALESCE(ac.affinity, 0) != -1
|
||||
GROUP BY r.company_key, r.company
|
||||
ORDER BY r.company_key, n DESC, r.company""", (agent_id,))
|
||||
found: dict[str, tuple[list[str], list[int], int]] = {}
|
||||
for r in rows:
|
||||
names, counts, _ = found.setdefault(
|
||||
r["key"], ([r["own"]] if r["own"] else [], [], int(r["affinity"])))
|
||||
if r["name"] and r["name"] not in names:
|
||||
names.append(r["name"])
|
||||
counts.append(int(r["n"]))
|
||||
return [(key, names, sum(counts), affinity)
|
||||
for key, (names, counts, affinity) in found.items()]
|
||||
|
||||
def open_places(self) -> list[tuple[str, str, int]]:
|
||||
"""`(place, country, open roles)` for every place line an open role
|
||||
carries, what `places.suggest` reads."""
|
||||
return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows(
|
||||
"SELECT place, country, COUNT(*) AS n FROM roles "
|
||||
"WHERE closed_at IS NULL AND place != '' GROUP BY place, country")]
|
||||
|
||||
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
||||
"""Per subscription id, its open roles counted by `country`."""
|
||||
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
|
||||
|
||||
+161
-27
@@ -11,12 +11,12 @@ from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app import jobs
|
||||
from app import jobs, places
|
||||
from app.models import (
|
||||
AFFINITY, Agent, Application, Company, Gate, Limits, Prompts, Role, RoleRow,
|
||||
AFFINITY, LEVELS, Agent, Application, Company, Gate, Limits, Prompts, Role, RoleRow,
|
||||
HandReason, ScoredRole, Status, Subscription, Weights, company_key,
|
||||
)
|
||||
from app.storage import Store
|
||||
@@ -59,6 +59,76 @@ class VoteBody(BaseModel):
|
||||
agent: int | None = None
|
||||
|
||||
|
||||
def _feed_filters(
|
||||
store: StoreDep,
|
||||
q: str = "",
|
||||
company: str = "",
|
||||
place: str = "",
|
||||
function: str = "",
|
||||
remote: str = "",
|
||||
days: int | None = None,
|
||||
show_muted: bool = False,
|
||||
include_read: bool = True,
|
||||
tags: list[str] = Query(default=[]),
|
||||
tag_mode: Literal["any", "all"] = "any",
|
||||
agent: int | None = None,
|
||||
order: Literal["score", "newest"] = "score",
|
||||
gated: bool = False,
|
||||
judged: Literal["", "liked", "unjudged", "passed"] = "",
|
||||
pay: Literal["", "stated", "normalised", "stated_only"] = "",
|
||||
company_keys: list[str] = Query(default=[]),
|
||||
tokens: list[str] = Query(default=[], alias="places"),
|
||||
seniority: list[str] = Query(default=[]),
|
||||
seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""The feed's filters, as `Store.roles_page`, `count_roles` and
|
||||
`role_facets` take them, read from the query of `GET /api/roles` and
|
||||
`GET /api/roles/facets` alike. A place token that names nothing and a
|
||||
level outside `LEVELS` are a 422 sentence, not an empty feed."""
|
||||
for token in tokens:
|
||||
if not places.known(token):
|
||||
raise HTTPException(422, f"No place is known as {token!r}.")
|
||||
for level in seniority:
|
||||
if level not in LEVELS:
|
||||
raise HTTPException(422, f"Seniority is one of {', '.join(LEVELS)}, not {level!r}.")
|
||||
return {
|
||||
"q": q or None, "company": company or None, "place": place or None,
|
||||
"function": function or None, "remote": remote or None, "days": days,
|
||||
"hide_muted": not show_muted, "include_read": include_read, "tags": tags,
|
||||
"tag_mode": tag_mode, "agent": _agent_id(store, agent),
|
||||
"by_score": agent is not None and order == "score", "gated": gated,
|
||||
"judged": judged or None, "pay": pay or None, "company_keys": company_keys,
|
||||
"places": tokens, "seniority": seniority, "seniority_unknown": seniority_unknown,
|
||||
"pay_unknown": pay_unknown}
|
||||
|
||||
|
||||
FeedFilters = Annotated[dict[str, Any], Depends(_feed_filters)]
|
||||
|
||||
|
||||
class FacetCount(BaseModel):
|
||||
"""One value of a facet and how many roles hold it; `value` "" is unknown."""
|
||||
|
||||
value: str
|
||||
count: int
|
||||
|
||||
|
||||
class Facets(BaseModel):
|
||||
seniority: list[FacetCount]
|
||||
pay: list[FacetCount]
|
||||
|
||||
|
||||
@router.get("/roles/facets")
|
||||
def role_facets(store: StoreDep, filters: FeedFilters) -> Facets:
|
||||
"""How many roles the feed with these parameters holds at each seniority
|
||||
level and each pay state, each facet counted with its own filter left
|
||||
out (`Store.role_facets`). Declared before `/roles/{uid}`, which would
|
||||
otherwise read `facets` as a uid."""
|
||||
found = store.role_facets(**filters)
|
||||
return Facets(**{name: [FacetCount(value=v, count=n) for v, n in counts]
|
||||
for name, counts in found.items()})
|
||||
|
||||
|
||||
@router.get("/roles/{uid}")
|
||||
def get_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole:
|
||||
"""One role, open or closed, for the read page, with `agent`'s vote and
|
||||
@@ -126,21 +196,7 @@ class RolesPage(BaseModel):
|
||||
@router.get("/roles")
|
||||
def list_roles(
|
||||
store: StoreDep,
|
||||
q: str = "",
|
||||
company: str = "",
|
||||
place: str = "",
|
||||
function: str = "",
|
||||
remote: str = "",
|
||||
days: int | None = None,
|
||||
show_muted: bool = False,
|
||||
include_read: bool = True,
|
||||
tags: list[str] = Query(default=[]),
|
||||
tag_mode: Literal["any", "all"] = "any",
|
||||
agent: int | None = None,
|
||||
order: Literal["score", "newest"] = "score",
|
||||
gated: bool = False,
|
||||
judged: Literal["", "liked", "unjudged", "passed"] = "",
|
||||
pay: Literal["", "stated", "normalised", "stated_only"] = "",
|
||||
filters: FeedFilters,
|
||||
offset: Annotated[int, Query(ge=0)] = 0,
|
||||
limit: Annotated[int, Query(ge=1, le=1000)] = 60,
|
||||
) -> RolesPage:
|
||||
@@ -161,15 +217,16 @@ def list_roles(
|
||||
`?tags=a&tags=b`, kept to a role with any of them or, with
|
||||
`tag_mode=all`, every one. `pay` keeps only roles whose `salary_state`
|
||||
is stated or normalised, normalised only, or stated but not normalised
|
||||
(`Store.PAY`)."""
|
||||
agent_id = _agent_id(store, agent)
|
||||
by_score = agent is not None and order == "score"
|
||||
filters: dict[str, Any] = {
|
||||
"q": q or None, "company": company or None, "place": place or None,
|
||||
"function": function or None, "remote": remote or None, "days": days,
|
||||
"hide_muted": not show_muted, "include_read": include_read, "tags": tags,
|
||||
"tag_mode": tag_mode, "agent": agent_id, "by_score": by_score,
|
||||
"gated": gated, "judged": judged or None, "pay": pay or None}
|
||||
(`Store.PAY`); `pay_unknown` lets the roles that state no pay through
|
||||
as well.
|
||||
|
||||
`company` and `place` are substrings, kept for links from other pages.
|
||||
The rail's fields are `company_keys` (repeated, exact keys),
|
||||
`places` (repeated place tokens, `app.places`: `country:DK`,
|
||||
`region:skane`, `place:malmo`, matched on whole folded words) and
|
||||
`seniority` (repeated levels), each several values joined by OR. A role
|
||||
with no level passes the seniority filter unless `seniority_unknown` is
|
||||
false. The parameters are read by `_feed_filters`."""
|
||||
return RolesPage(items=store.roles_page(limit=limit, offset=offset, **filters),
|
||||
total=store.count_roles(**filters))
|
||||
|
||||
@@ -198,6 +255,83 @@ def list_filters(store: StoreDep, show_muted: bool = False,
|
||||
for tag, count, unread in store.tag_counts(hidden_for, show_muted)])
|
||||
|
||||
|
||||
# ------------------------------------------------------------ suggestions
|
||||
|
||||
class CompanySuggestion(BaseModel):
|
||||
"""A company for the rail's Company field: its key, the name shown, its
|
||||
open roles and the agent's affinity."""
|
||||
|
||||
key: str
|
||||
name: str
|
||||
count: int
|
||||
affinity: int
|
||||
|
||||
|
||||
class PlaceSuggestion(BaseModel):
|
||||
"""A place token for the rail's Place field (`places.Suggestion`)."""
|
||||
|
||||
token: str
|
||||
label: str
|
||||
kind: Literal["country", "region", "place"]
|
||||
count: int
|
||||
|
||||
|
||||
class PlaceLabel(BaseModel):
|
||||
token: str
|
||||
label: str
|
||||
|
||||
|
||||
def _starts_a_word(text: str, folded: str) -> bool:
|
||||
"""`text` starts a word, or a run of words, of `folded`: `nor` finds `novo
|
||||
nordisk`, `novo nor` too. Both are folded already."""
|
||||
return f" {text}" in f" {folded}"
|
||||
|
||||
|
||||
@router.get("/suggest/companies")
|
||||
def suggest_companies(
|
||||
store: StoreDep,
|
||||
q: str = "",
|
||||
agent: int | None = None,
|
||||
limit: Annotated[int, Query(ge=1, le=50)] = 12,
|
||||
) -> list[CompanySuggestion]:
|
||||
"""Companies with open roles whose name has a word that starts with `q`,
|
||||
folded as places are (`places.fold`), so `orsted` finds Ørsted and
|
||||
`kobenhavn` finds a name with København in it. By open roles, most
|
||||
first. The companies `agent` (the default agent when absent) hid are
|
||||
left out, as the feed leaves them out. An empty `q` suggests nothing."""
|
||||
text = places.fold(q)
|
||||
if not text:
|
||||
return []
|
||||
found = [CompanySuggestion(key=key, name=names[0] if names else key, count=count,
|
||||
affinity=affinity)
|
||||
for key, names, count, affinity in store.open_companies(_agent_id(store, agent))
|
||||
if any(_starts_a_word(text, places.fold(name)) for name in names)]
|
||||
found.sort(key=lambda c: (-c.count, c.name.casefold()))
|
||||
return found[:limit]
|
||||
|
||||
|
||||
@router.get("/suggest/places")
|
||||
def suggest_places(
|
||||
store: StoreDep,
|
||||
q: str = "",
|
||||
limit: Annotated[int, Query(ge=1, le=50)] = 12,
|
||||
) -> list[PlaceSuggestion]:
|
||||
"""Place tokens for the text typed so far: countries, then regions, then
|
||||
towns, each by open roles (`places.suggest`, over every open role's place
|
||||
line). An empty `q` suggests nothing."""
|
||||
return [PlaceSuggestion(token=s.token, label=s.label, kind=s.kind, count=s.count)
|
||||
for s in places.suggest(q, store.open_places(), limit)]
|
||||
|
||||
|
||||
@router.get("/places/labels")
|
||||
def place_labels(tokens: list[str] = Query(default=[])) -> list[PlaceLabel]:
|
||||
"""The chip text of each token read from the address, in the order
|
||||
asked: `country:DK` is Denmark, `region:skane` Skåne. A token that
|
||||
names nothing is left out, so the page drops its chip rather than send
|
||||
it to `/api/roles` for a 422."""
|
||||
return [PlaceLabel(token=t, label=places.label(t)) for t in tokens if places.known(t)]
|
||||
|
||||
|
||||
# ------------------------------------------------- roles: text and summary
|
||||
|
||||
#: What the read page toasts when a posting page gave nothing readable, by
|
||||
|
||||
Reference in new issue
Block a user