From a91b4e99e0d3646ca65299c84b12c582fad6e047 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 22:17:12 +0200 Subject: [PATCH] Filter the feed by company keys, place tokens and seniority, and suggest companies and places Migration 17 adds roles.place_fold and fills seniority from the title where extract has not read the role. Every connection registers place_match, so the Place filter and the place gate match whole folded words: Lund no longer finds Billund. GET /api/roles takes company_keys, places, seniority, seniority_unknown and pay_unknown; new routes suggest companies and places, label place tokens and count the seniority and pay facets. Co-Authored-By: Claude --- app/models.py | 10 +- app/storage/schema.py | 43 ++++++++ app/storage/store.py | 211 ++++++++++++++++++++++++++++++++++----- app/web/api.py | 188 ++++++++++++++++++++++++++++++----- tests/test_storage.py | 224 ++++++++++++++++++++++++++++++++++++++++-- tests/test_web.py | 108 +++++++++++++++++++- 6 files changed, 724 insertions(+), 60 deletions(-) diff --git a/app/models.py b/app/models.py index f4b3afc..84e72b4 100644 --- a/app/models.py +++ b/app/models.py @@ -632,6 +632,8 @@ def remote_of(place: str) -> str: Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager", "director", "exec"] SENIORITIES: tuple[str, ...] = get_args(Seniority) +#: The levels a seniority filter may name: `SENIORITIES` without the unknown. +LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level) #: Whether a posting says it sponsors a work permit. '' is "it does not say". Permit = Literal["", "sponsors", "no_sponsorship"] @@ -891,7 +893,9 @@ class RoleRow(Model): characters. The pay fields are what the row's badge reads: the three normalised columns, the line the board stated, and `salary_state`, derived as on `Role`. `original_url` is one short string, kept so the - `o` key opens a selected row's posting without opening the reader.""" + `o` key opens a selected row's posting without opening the reader. + `seniority` is the title's level until extract reads the posting, '' + when neither says.""" uid: str company_key: str = "" @@ -901,6 +905,7 @@ class RoleRow(Model): country: str = "" remote: Remote = "" function: str = "" + seniority: str = "" salary_min: int | None = None salary_max: int | None = None currency: str = "" @@ -998,7 +1003,8 @@ class Application(Model): # ------------------------------------------------------------------ agents #: The role columns a gate can name. `company` matches on the company key; -#: `place` is a case-insensitive substring, like the place filter; the rest are +#: `place` takes the Place filter's tokens (`app.places`: `country:DK`, +#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are #: case-insensitive equality, except `salary_min` (at least the value) and #: `languages` (every language the role requires is one of the values). GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min", diff --git a/app/storage/schema.py b/app/storage/schema.py index ff3353c..662b3b5 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -17,7 +17,9 @@ import logging import sqlite3 from collections.abc import Callable +from app import places from app.models import country_of, posted_at, posted_host +from app.seniority import from_title log = logging.getLogger(__name__) @@ -42,12 +44,16 @@ CREATE TABLE IF NOT EXISTS roles ( -- require the country: "UK | Remote" is not remote from Berlin. place TEXT NOT NULL DEFAULT '', place_key TEXT NOT NULL DEFAULT '', + -- places.fold(place): what the Place filter and the place gate match + -- whole words against, through the place_match function (app/places.py). + place_fold TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', remote TEXT NOT NULL DEFAULT '' CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')), salary_min INTEGER, salary_max INTEGER, currency TEXT NOT NULL DEFAULT '', + -- seniority.from_title(title) at ingest, until extract reads the posting. seniority TEXT NOT NULL DEFAULT '', function TEXT NOT NULL DEFAULT '', -- Whether the posting says it sponsors a work permit, and the languages @@ -446,6 +452,9 @@ ALTER TABLE companies ADD COLUMN careers_opened_at TEXT NOT NULL DEFAULT ''; (16, "companies.hand_reason: why a company is read by hand", """ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; """), + # A Python step: the fold and the title classifier are Python, as in 11. + (17, "roles.place_fold for the whole-word place filter, and roles.seniority from " + "the title where extract has not read the role", lambda conn: _fold_places(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -523,6 +532,40 @@ def _recompute_country(conn: sqlite3.Connection) -> None: len(updates), len(rows)) +def _fold_places(conn: sqlite3.Connection) -> None: + """Migration 17's step. Adds `roles.place_fold` and fills it with + `places.fold(place)` for every role, as `Store.upsert_role` writes it for + a new one, and fills an empty `seniority` from `seniority.from_title` on + every role `extract` has not read (it overwrites the title's level when it + does, see `Store.set_extracted`). + + Each distinct place and title is folded or classified once: 34,538 roles + hold about 5,600 places. The rows are written with one `executemany` per + column inside `_apply`'s transaction, and only those with something to + write.""" + conn.execute("ALTER TABLE roles ADD COLUMN place_fold TEXT NOT NULL DEFAULT ''") + rows = conn.execute( + "SELECT uid, place, title, seniority, extracted_at FROM roles").fetchall() + folds: dict[str, str] = {} + levels: dict[str, str] = {} + place_updates, level_updates = [], [] + for uid, place, title, level, extracted_at in rows: + if place: + if place not in folds: + folds[place] = places.fold(place) + if folds[place]: + place_updates.append((folds[place], uid)) + if not level and extracted_at is None and title: + if title not in levels: + levels[title] = from_title(title) + if levels[title]: + level_updates.append((levels[title], uid)) + conn.executemany("UPDATE roles SET place_fold = ? WHERE uid = ?", place_updates) + conn.executemany("UPDATE roles SET seniority = ? WHERE uid = ?", level_updates) + log.info("migration 17: folded %d places over %d roles, a level from the title for %d", + len(folds), len(place_updates), len(level_updates)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/storage/store.py b/app/storage/store.py index c3f880e..8ccd69e 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -20,26 +20,30 @@ from uuid import uuid4 import numpy as np from desk.models import Item +from app import places from app.models import ( - AFFINITY, HAND_REASONS, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, - Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, Subscription, Weights, - check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host, - remote_of, title_key, + AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application, + ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, + Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key, + posted_at, posted_host, remote_of, title_key, ) +from app.seniority import from_title from app.storage import schema #: Text columns a plugin may fill from `Item.raw` under the same name. An #: empty value never overwrites a filled column, so an upsert from a board that #: says less than `extract` found keeps what `extract` found. `function` is #: not here: it is not a board field (see the note by `keep_filled` below). -_RAW_TEXT = ("place", "country", "currency", "seniority") +#: Nor is `seniority`, which is the title's level unless a board states one. +_RAW_TEXT = ("place", "country", "currency") _RAW_INT = ("salary_min", "salary_max") #: What `extract` may fill. Only an empty column is written: a board's own #: structured field wins over what a model read out of the prose, and the next -#: upsert from that board would put it back anyway. `function` is not here -#: either, for the opposite reason — see `set_extracted`. -_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "permit") +#: upsert from that board would put it back anyway. `function` and +#: `seniority` are not here either, for the opposite reason — see +#: `set_extracted`. +_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit") #: How many replies may leave a role unanswered before `extract` stops #: offering it. newsdesk's number, for newsdesk's reason: a row the model @@ -72,6 +76,15 @@ PAY: dict[str, str] = { "stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)", } +#: A role that states no pay at all, what `pay_unknown` lets through the Pay +#: filter, and the `""` of its facet. +_NO_PAY = "(salary_min IS NULL AND salary_max IS NULL AND salary_text = '')" + +#: `salary_state` in SQL, what the Pay facet counts by: 'normalised', +#: 'stated' (a line and no figure) or ''. +_PAY_STATE = ("CASE WHEN salary_min IS NOT NULL OR salary_max IS NOT NULL THEN 'normalised' " + "WHEN salary_text != '' THEN 'stated' ELSE '' END") + #: A ranking's score as `Store.roles` serves and orders it: halved by a pass, #: lifted a fifth by a like, capped at 1. Applied at read time, so a vote #: moves the role at once and the stored score stays what `score` computed. @@ -173,8 +186,12 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]: floor = min(_int_or_none(v) or 0 for v in gate.values) return f"{column} IS NULL", f"{column} >= ?", [floor] if gate.field == "place": - likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0" - return "r.place = ''", f"({likes})", [_like(v) for v in values] + # Whole folded words, as the feed's Place filter: `lund` is not in + # `Billund`. A value is a place token, bare text a `place:`. + clauses = [places.match_sql(v, place="r.place_fold", country="r.country") + for v in values] + match = " OR ".join(sql for sql, _ in clauses) or "0" + return "r.place = ''", f"({match})", [p for _, ps in clauses for p in ps] if gate.field == "company": keys = [company_key(v) for v in values] return ("r.company_key = ''", @@ -186,6 +203,20 @@ def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]: return f"{column} = ''", f"LOWER({column}) IN ({marks})", values +def _places_sql(tokens: Sequence[str]) -> tuple[str, list[Any]]: + """The Place filter over `roles`: any of the tokens, each a `place_match` + call (`places.match_sql`). ValueError for a token that names nothing, + which would otherwise match no role without saying why.""" + clauses, params = [], [] + for token in tokens: + if not places.known(token): + raise ValueError(f"{token!r} is not a place this app knows") + sql, values = places.match_sql(token, place="roles.place_fold", country="roles.country") + clauses.append(sql) + params += values + return f"({' OR '.join(clauses)})", params + + def _int_or_none(value: Any) -> int | None: if value is None or value == "" or isinstance(value, bool): return None @@ -242,9 +273,12 @@ class Store: def _connect(self) -> sqlite3.Connection: """A fresh connection. `foreign_keys` is off by default per connection; `busy_timeout` matches `pipeline/table.py`, which writes the same file; - `synchronous=NORMAL` is the standard WAL companion.""" + `synchronous=NORMAL` is the standard WAL companion. `place_match` is + registered on every one, since the place filter and the place gate + call it (`places.match_sql`).""" conn = sqlite3.connect(self.path) conn.row_factory = sqlite3.Row + places.register(conn) conn.execute("PRAGMA foreign_keys=ON") conn.execute("PRAGMA busy_timeout=30000") conn.execute("PRAGMA synchronous=NORMAL") @@ -302,6 +336,14 @@ class Store: `extract` (`extracted_at` is set), this upsert leaves `function` alone for good, whatever the plugin's raw guess now says — see `set_extracted` for the other half of the rule. + + `seniority` is the title's level (`seniority.from_title`) until + extract has read the role and extract's from then on, as `function` + is. The classifier is called here rather than in a plugin, so a role + from any source gets it, the way `country` comes from `place` here + too. A level the board states in `raw` is a board field and wins over + both. `place_fold` is written with `place`, under the same keep-filled + rule. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") @@ -336,6 +378,9 @@ class Store: row["remote"] = row["remote"] or remote_of(place) row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} row["function"] = str(raw.get("function") or "") + stated = str(raw.get("seniority") or "") + row["seniority"] = stated or from_title(item.title) + row["place_fold"] = places.fold(row["place"]) # A board's own pay line, one per pull like title or published_at — # not kept-filled like _RAW_TEXT: a board that stops stating pay # must be able to clear it, not leave a stale line behind. @@ -359,6 +404,13 @@ class Store: updates.append("function=CASE WHEN roles.extracted_at IS NULL " "THEN COALESCE(NULLIF(excluded.function, ''), roles.function) " "ELSE roles.function END") + # The title's level follows the title until extract has read the + # role, and is extract's from then on; a board's stated level wins. + updates.append("seniority=excluded.seniority" if stated else + "seniority=CASE WHEN roles.extracted_at IS NULL " + "THEN excluded.seniority ELSE roles.seniority END") + updates.append("place_fold=CASE WHEN excluded.place != '' " + "THEN excluded.place_fold ELSE roles.place_fold END") updates.append("closed_at=NULL") columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) @@ -492,7 +544,9 @@ class Store: tags: Sequence[str] = (), tag_mode: str = "any", agent: int | None = None, by_score: bool = True, gated: bool = False, judged: str | None = None, pay: str | None = None, - offset: int = 0) -> list[ScoredRole]: + company_keys: Sequence[str] = (), places: Sequence[str] = (), + seniority: Sequence[str] = (), seniority_unknown: bool = True, + pay_unknown: bool = False, offset: int = 0) -> list[ScoredRole]: """Newest first by `posted_at`. `company` and `place` are case-insensitive substrings of the columns a board wrote — not the key, so a search does not have to know the exact slug a company folds @@ -500,7 +554,15 @@ class Store: role must match every word in one or the other. `function` and `remote` match a gate column exactly. `days` keeps only roles posted within that many days of now. `pay` (a key of `PAY`) filters on - `salary_state`. + `salary_state`; with `pay_unknown` a role that states no pay passes + it too. + + `company_keys` keeps those exact companies. `places` keeps a role any + of the place tokens finds (`app.places`: `country:DK`, `region:skane`, + `place:malmo`), matched on whole folded words; a token that names + nothing is a ValueError. `seniority` keeps the roles at any of those + levels (`LEVELS`) and, with `seniority_unknown` (the default), those + with no level, which is most of them. An empty list is no filter. `agent` says whose votes, hidden companies and scores are meant. Each row carries that agent's `vote` and `voted_at`; `judged` (a key of @@ -535,7 +597,8 @@ class Store: company=company, place=place, open_only=open_only, q=q, function=function, remote=remote, days=days, hide_muted=hide_muted, include_read=include_read, tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated, - judged=judged, pay=pay) + judged=judged, pay=pay, company_keys=company_keys, places=places, + seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown) select = "SELECT roles.*, v.vote, v.voted_at" order = "roles.posted_at DESC, roles.uid" scored = agent is not None and by_score @@ -566,7 +629,9 @@ class Store: tags: Sequence[str] = (), tag_mode: str = "any", agent: int | None = None, by_score: bool = True, gated: bool = False, judged: str | None = None, pay: str | None = None, - offset: int = 0) -> list[RoleRow]: + company_keys: Sequence[str] = (), places: Sequence[str] = (), + seniority: Sequence[str] = (), seniority_unknown: bool = True, + pay_unknown: bool = False, offset: int = 0) -> list[RoleRow]: """What `GET /api/roles` serves: the same rows `roles` would, in the same order, with the same filters and paging, but only the fields a row draws (see `RoleRow`) — never `description`, `feed_text`, `raw` @@ -581,10 +646,11 @@ class Store: company=company, place=place, open_only=open_only, q=q, function=function, remote=remote, days=days, hide_muted=hide_muted, include_read=include_read, tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated, - judged=judged, pay=pay) + judged=judged, pay=pay, company_keys=company_keys, places=places, + seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown) select = ( "SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, " - "roles.country, roles.remote, roles.function, roles.salary_min, " + "roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, " "roles.salary_max, roles.currency, roles.salary_text, roles.original_url, " "roles.posted_at, roles.read_at, " "v.vote, v.voted_at, " @@ -617,22 +683,64 @@ class Store: include_read: bool = True, tags: Sequence[str] = (), tag_mode: str = "any", agent: int | None = None, by_score: bool = True, gated: bool = False, judged: str | None = None, - pay: str | None = None) -> int: + pay: str | None = None, company_keys: Sequence[str] = (), + places: Sequence[str] = (), seniority: Sequence[str] = (), + seniority_unknown: bool = True, pay_unknown: bool = False) -> int: """How many roles `roles` would list with the same filters and no limit.""" sql, params = self._roles_from( company=company, place=place, open_only=open_only, q=q, function=function, remote=remote, days=days, hide_muted=hide_muted, include_read=include_read, tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated, - judged=judged, pay=pay) + judged=judged, pay=pay, company_keys=company_keys, places=places, + seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown) row = self._row(f"SELECT COUNT(*) AS n {sql}", params) return int(row["n"]) if row else 0 + def role_facets(self, company: str | None = None, place: str | None = None, + open_only: bool = True, q: str | None = None, + function: str | None = None, remote: str | None = None, + days: int | None = None, hide_muted: bool = False, + include_read: bool = True, tags: Sequence[str] = (), + tag_mode: str = "any", agent: int | None = None, by_score: bool = True, + gated: bool = False, judged: str | None = None, + pay: str | None = None, company_keys: Sequence[str] = (), + places: Sequence[str] = (), seniority: Sequence[str] = (), + seniority_unknown: bool = True, + pay_unknown: bool = False) -> dict[str, list[tuple[str, int]]]: + """How many roles `roles` would list at each seniority level and each + pay state, each counted with every other filter and its own left + out, so the rail can say what a click would add. `seniority` is every + level of `LEVELS` in order and then `""`, no level; `pay` is + `salary_state`'s 'normalised' and 'stated' (a line with no figure) + and then `""`, no pay stated. A value no role has counts 0.""" + shared: dict[str, Any] = { + "company": company, "place": place, "open_only": open_only, "q": q, + "function": function, "remote": remote, "days": days, "hide_muted": hide_muted, + "include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent, + "by_score": by_score, "gated": gated, "judged": judged, + "company_keys": company_keys, "places": places} + by_level, level_params = self._roles_from( + **shared, pay=pay, pay_unknown=pay_unknown) + by_pay, pay_params = self._roles_from( + **shared, seniority=seniority, seniority_unknown=seniority_unknown) + levels = {r["value"]: int(r["n"]) for r in self._rows( + f"SELECT roles.seniority AS value, COUNT(*) AS n {by_level} " + "GROUP BY roles.seniority", level_params)} + pays = {r["value"]: int(r["n"]) for r in self._rows( + f"SELECT {_PAY_STATE} AS value, COUNT(*) AS n {by_pay} GROUP BY value", + pay_params)} + return {"seniority": [(v, levels.get(v, 0)) for v in (*LEVELS, "")], + "pay": [(v, pays.get(v, 0)) for v in ("normalised", "stated", "")]} + def _roles_from(self, *, company: str | None, place: str | None, open_only: bool, q: str | None, function: str | None, remote: str | None, days: int | None, hide_muted: bool, include_read: bool, tags: Sequence[str], tag_mode: str, agent: int | None, by_score: bool, gated: bool, judged: str | None, - pay: str | None = None) -> tuple[str, list[Any]]: + pay: str | None = None, company_keys: Sequence[str] = (), + places: Sequence[str] = (), seniority: Sequence[str] = (), + seniority_unknown: bool = True, + pay_unknown: bool = False) -> tuple[str, list[Any]]: """The `FROM ... WHERE ...` that `roles` and `count_roles` share, with its parameters in order. The agent's votes are always joined as `v` (see `_WITH_VOTES`); its scores as `s` only when ordering by them.""" @@ -663,7 +771,16 @@ class Store: if pay: if pay not in PAY: raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}") - where.append(PAY[pay]) + where.append(f"({PAY[pay]} OR {_NO_PAY})" if pay_unknown else PAY[pay]) + if company_keys: + where.append(f"company_key IN ({', '.join('?' for _ in company_keys)})") + params += list(company_keys) + if seniority: + if unknown := sorted(set(seniority) - set(LEVELS)): + raise ValueError(f"seniority is {', '.join(LEVELS)}, not {unknown[0]!r}") + levels = f"roles.seniority IN ({', '.join('?' for _ in seniority)})" + where.append(f"({levels} OR roles.seniority = '')" if seniority_unknown else levels) + params += list(seniority) if open_only: where.append("closed_at IS NULL") if hide_muted and agent is not None: @@ -686,6 +803,11 @@ class Store: params += list(tags) else: raise ValueError(f"tag_mode is any or all, not {tag_mode!r}") + if places: + # Last, so the cheaper terms have cut the rows it is called on. + clause, values = _places_sql(places) + where.append(clause) + params += values sql = f"FROM roles {_WITH_VOTES}" joins: list[Any] = [agent] if agent is not None and by_score: @@ -824,7 +946,7 @@ class Store: Only into empty columns, and only a non-empty value: a board's own field is never replaced and nothing is ever blanked. `place` brings - `place_key` with it. Returns how many columns were filled, or None + `place_key` and `place_fold` with it. Returns how many columns were filled, or None for an unknown uid. Values are trusted to be in vocabulary; the job's `apply` is where a reply is checked. @@ -832,7 +954,10 @@ class Store: the `ats` classifier's keyword guess, not a board field, so a non-empty answer overwrites it outright rather than only filling an empty column — see `upsert_role`'s docstring for the other half of - the rule that then keeps a later pull from putting the guess back.""" + the rule that then keeps a later pull from putting the guess back. + `seniority` is the other: `upsert_role` fills it from the title, so a + non-empty answer replaces it, unless the board stated a level in + `raw`.""" with self.transaction() as conn: row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone() if row is None: @@ -845,6 +970,9 @@ class Store: function = str(fields.get("function") or "") if function: sets["function"] = function + level = str(fields.get("seniority") or "") + if level and not json.loads(row["raw"] or "{}").get("seniority"): + sets["seniority"] = level for name in _RAW_INT: number = _int_or_none(fields.get(name)) if number is not None and row[name] is None: @@ -855,6 +983,7 @@ class Store: filled = len(sets) if "place" in sets: sets["place_key"] = place_key(sets["place"]) + sets["place_fold"] = places.fold(sets["place"]) sets["extracted_at"] = self.now() conn.execute( f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?", @@ -1273,6 +1402,42 @@ class Store: "SELECT company_key, COUNT(*) AS n FROM roles " "WHERE closed_at IS NULL GROUP BY company_key")} + def open_companies(self, agent_id: int | None = None + ) -> list[tuple[str, list[str], int, int]]: + """`(company_key, names, open roles, affinity)` for every company with + an open role, what the Company field suggests from. `names` are the + company row's name and then every spelling its roles carry, commonest + first, so a search finds a company by any of them. A company + `agent_id` hid is left out; `affinity` is that agent's, 0 without + one.""" + rows = self._rows( + """SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n, + c.name AS own, COALESCE(ac.affinity, 0) AS affinity + FROM roles r + LEFT JOIN companies c ON c.key = r.company_key + LEFT JOIN agent_companies ac ON ac.company_key = r.company_key + AND ac.agent_id = ? + WHERE r.closed_at IS NULL AND r.company_key != '' + AND COALESCE(ac.affinity, 0) != -1 + GROUP BY r.company_key, r.company + ORDER BY r.company_key, n DESC, r.company""", (agent_id,)) + found: dict[str, tuple[list[str], list[int], int]] = {} + for r in rows: + names, counts, _ = found.setdefault( + r["key"], ([r["own"]] if r["own"] else [], [], int(r["affinity"]))) + if r["name"] and r["name"] not in names: + names.append(r["name"]) + counts.append(int(r["n"])) + return [(key, names, sum(counts), affinity) + for key, (names, counts, affinity) in found.items()] + + def open_places(self) -> list[tuple[str, str, int]]: + """`(place, country, open roles)` for every place line an open role + carries, what `places.suggest` reads.""" + return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows( + "SELECT place, country, COUNT(*) AS n FROM roles " + "WHERE closed_at IS NULL AND place != '' GROUP BY place, country")] + def open_countries_by_subscription(self) -> dict[int, dict[str, int]]: """Per subscription id, its open roles counted by `country`.""" rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles diff --git a/app/web/api.py b/app/web/api.py index 0aadefb..a573b70 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -11,12 +11,12 @@ from __future__ import annotations from typing import Annotated, Any, Literal -from fastapi import APIRouter, HTTPException, Query, Response +from fastapi import APIRouter, Depends, HTTPException, Query, Response from pydantic import BaseModel, Field -from app import jobs +from app import jobs, places from app.models import ( - AFFINITY, Agent, Application, Company, Gate, Limits, Prompts, Role, RoleRow, + AFFINITY, LEVELS, Agent, Application, Company, Gate, Limits, Prompts, Role, RoleRow, HandReason, ScoredRole, Status, Subscription, Weights, company_key, ) from app.storage import Store @@ -59,6 +59,76 @@ class VoteBody(BaseModel): agent: int | None = None +def _feed_filters( + store: StoreDep, + q: str = "", + company: str = "", + place: str = "", + function: str = "", + remote: str = "", + days: int | None = None, + show_muted: bool = False, + include_read: bool = True, + tags: list[str] = Query(default=[]), + tag_mode: Literal["any", "all"] = "any", + agent: int | None = None, + order: Literal["score", "newest"] = "score", + gated: bool = False, + judged: Literal["", "liked", "unjudged", "passed"] = "", + pay: Literal["", "stated", "normalised", "stated_only"] = "", + company_keys: list[str] = Query(default=[]), + tokens: list[str] = Query(default=[], alias="places"), + seniority: list[str] = Query(default=[]), + seniority_unknown: bool = True, + pay_unknown: bool = False, +) -> dict[str, Any]: + """The feed's filters, as `Store.roles_page`, `count_roles` and + `role_facets` take them, read from the query of `GET /api/roles` and + `GET /api/roles/facets` alike. A place token that names nothing and a + level outside `LEVELS` are a 422 sentence, not an empty feed.""" + for token in tokens: + if not places.known(token): + raise HTTPException(422, f"No place is known as {token!r}.") + for level in seniority: + if level not in LEVELS: + raise HTTPException(422, f"Seniority is one of {', '.join(LEVELS)}, not {level!r}.") + return { + "q": q or None, "company": company or None, "place": place or None, + "function": function or None, "remote": remote or None, "days": days, + "hide_muted": not show_muted, "include_read": include_read, "tags": tags, + "tag_mode": tag_mode, "agent": _agent_id(store, agent), + "by_score": agent is not None and order == "score", "gated": gated, + "judged": judged or None, "pay": pay or None, "company_keys": company_keys, + "places": tokens, "seniority": seniority, "seniority_unknown": seniority_unknown, + "pay_unknown": pay_unknown} + + +FeedFilters = Annotated[dict[str, Any], Depends(_feed_filters)] + + +class FacetCount(BaseModel): + """One value of a facet and how many roles hold it; `value` "" is unknown.""" + + value: str + count: int + + +class Facets(BaseModel): + seniority: list[FacetCount] + pay: list[FacetCount] + + +@router.get("/roles/facets") +def role_facets(store: StoreDep, filters: FeedFilters) -> Facets: + """How many roles the feed with these parameters holds at each seniority + level and each pay state, each facet counted with its own filter left + out (`Store.role_facets`). Declared before `/roles/{uid}`, which would + otherwise read `facets` as a uid.""" + found = store.role_facets(**filters) + return Facets(**{name: [FacetCount(value=v, count=n) for v, n in counts] + for name, counts in found.items()}) + + @router.get("/roles/{uid}") def get_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole: """One role, open or closed, for the read page, with `agent`'s vote and @@ -126,21 +196,7 @@ class RolesPage(BaseModel): @router.get("/roles") def list_roles( store: StoreDep, - q: str = "", - company: str = "", - place: str = "", - function: str = "", - remote: str = "", - days: int | None = None, - show_muted: bool = False, - include_read: bool = True, - tags: list[str] = Query(default=[]), - tag_mode: Literal["any", "all"] = "any", - agent: int | None = None, - order: Literal["score", "newest"] = "score", - gated: bool = False, - judged: Literal["", "liked", "unjudged", "passed"] = "", - pay: Literal["", "stated", "normalised", "stated_only"] = "", + filters: FeedFilters, offset: Annotated[int, Query(ge=0)] = 0, limit: Annotated[int, Query(ge=1, le=1000)] = 60, ) -> RolesPage: @@ -161,15 +217,16 @@ def list_roles( `?tags=a&tags=b`, kept to a role with any of them or, with `tag_mode=all`, every one. `pay` keeps only roles whose `salary_state` is stated or normalised, normalised only, or stated but not normalised - (`Store.PAY`).""" - agent_id = _agent_id(store, agent) - by_score = agent is not None and order == "score" - filters: dict[str, Any] = { - "q": q or None, "company": company or None, "place": place or None, - "function": function or None, "remote": remote or None, "days": days, - "hide_muted": not show_muted, "include_read": include_read, "tags": tags, - "tag_mode": tag_mode, "agent": agent_id, "by_score": by_score, - "gated": gated, "judged": judged or None, "pay": pay or None} + (`Store.PAY`); `pay_unknown` lets the roles that state no pay through + as well. + + `company` and `place` are substrings, kept for links from other pages. + The rail's fields are `company_keys` (repeated, exact keys), + `places` (repeated place tokens, `app.places`: `country:DK`, + `region:skane`, `place:malmo`, matched on whole folded words) and + `seniority` (repeated levels), each several values joined by OR. A role + with no level passes the seniority filter unless `seniority_unknown` is + false. The parameters are read by `_feed_filters`.""" return RolesPage(items=store.roles_page(limit=limit, offset=offset, **filters), total=store.count_roles(**filters)) @@ -198,6 +255,83 @@ def list_filters(store: StoreDep, show_muted: bool = False, for tag, count, unread in store.tag_counts(hidden_for, show_muted)]) +# ------------------------------------------------------------ suggestions + +class CompanySuggestion(BaseModel): + """A company for the rail's Company field: its key, the name shown, its + open roles and the agent's affinity.""" + + key: str + name: str + count: int + affinity: int + + +class PlaceSuggestion(BaseModel): + """A place token for the rail's Place field (`places.Suggestion`).""" + + token: str + label: str + kind: Literal["country", "region", "place"] + count: int + + +class PlaceLabel(BaseModel): + token: str + label: str + + +def _starts_a_word(text: str, folded: str) -> bool: + """`text` starts a word, or a run of words, of `folded`: `nor` finds `novo + nordisk`, `novo nor` too. Both are folded already.""" + return f" {text}" in f" {folded}" + + +@router.get("/suggest/companies") +def suggest_companies( + store: StoreDep, + q: str = "", + agent: int | None = None, + limit: Annotated[int, Query(ge=1, le=50)] = 12, +) -> list[CompanySuggestion]: + """Companies with open roles whose name has a word that starts with `q`, + folded as places are (`places.fold`), so `orsted` finds Ørsted and + `kobenhavn` finds a name with København in it. By open roles, most + first. The companies `agent` (the default agent when absent) hid are + left out, as the feed leaves them out. An empty `q` suggests nothing.""" + text = places.fold(q) + if not text: + return [] + found = [CompanySuggestion(key=key, name=names[0] if names else key, count=count, + affinity=affinity) + for key, names, count, affinity in store.open_companies(_agent_id(store, agent)) + if any(_starts_a_word(text, places.fold(name)) for name in names)] + found.sort(key=lambda c: (-c.count, c.name.casefold())) + return found[:limit] + + +@router.get("/suggest/places") +def suggest_places( + store: StoreDep, + q: str = "", + limit: Annotated[int, Query(ge=1, le=50)] = 12, +) -> list[PlaceSuggestion]: + """Place tokens for the text typed so far: countries, then regions, then + towns, each by open roles (`places.suggest`, over every open role's place + line). An empty `q` suggests nothing.""" + return [PlaceSuggestion(token=s.token, label=s.label, kind=s.kind, count=s.count) + for s in places.suggest(q, store.open_places(), limit)] + + +@router.get("/places/labels") +def place_labels(tokens: list[str] = Query(default=[])) -> list[PlaceLabel]: + """The chip text of each token read from the address, in the order + asked: `country:DK` is Denmark, `region:skane` Skåne. A token that + names nothing is left out, so the page drops its chip rather than send + it to `/api/roles` for a 422.""" + return [PlaceLabel(token=t, label=places.label(t)) for t in tokens if places.known(t)] + + # ------------------------------------------------- roles: text and summary #: What the read page toasts when a posting page gave nothing readable, by diff --git a/tests/test_storage.py b/tests/test_storage.py index 2c13024..066d927 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -14,8 +14,8 @@ from desk.models import Item import numpy as np from app.models import ( - Application, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at, - posted_host, title_key, + LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key, + posted_at, posted_host, title_key, ) from app.storage.store import pack from app.storage import LATEST, Store @@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 16 + assert LATEST == 17 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -88,9 +88,15 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]: return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")] +def _undo_v17(conn: sqlite3.Connection) -> None: + """A fresh file put back to the v16 shape: no folded place on a role.""" + conn.execute("ALTER TABLE roles DROP COLUMN place_fold") + + def _undo_v16(conn: sqlite3.Connection) -> None: """A fresh file put back to the v15 shape: no by-hand reason on a company.""" + _undo_v17(conn) conn.execute("ALTER TABLE companies DROP COLUMN hand_reason") @@ -105,7 +111,7 @@ def _undo_v15(conn: sqlite3.Connection) -> None: def _undo_v14(conn: sqlite3.Connection) -> None: """A fresh file put back to the v13 shape: an agent's intent and its cached vector on the agent row, no prompts, no prompt on a score. Every - downgrade below starts here, after `_undo_v15`, since v16 is the last + downgrade below starts here, after `_undo_v15`, since v17 is the last migration.""" _undo_v15(conn) conn.execute("DROP TABLE agent_prompts") @@ -153,7 +159,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated( migrated = Store(path) - assert migrated.migrations_applied == [15, 16] + assert migrated.migrations_applied == [15, 16, 17] acme = migrated.company("acme") assert acme is not None assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example") @@ -358,7 +364,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14, 15, 16] + assert migrated.migrations_applied == [14, 15, 16, 17] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -382,7 +388,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None migrated = Store(path) - assert migrated.migrations_applied == [16] + assert migrated.migrations_applied == [16, 17] acme = migrated.company("acme") assert acme is not None assert (acme.careers_url, acme.notes, acme.hand_reason) == ( @@ -390,6 +396,35 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None assert sorted(_columns(path, "companies")) == sorted(_columns(fresh, "companies")) +def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( + tmp_path: Path) -> None: + path = tmp_path / "v16.db" + store = Store(path) + plain = posting("plain", place="København K") # Senior ... in the title + extracted = posting("extracted", place="Malmö") + kept = posting("kept") + unknown = posting("unknown") + unknown.title = "Backend Engineer" + for item in (plain, extracted, kept, unknown): + store.upsert_role(item) + with sqlite3.connect(path) as conn: + _undo_v17(conn) + conn.execute("UPDATE roles SET seniority = ''") + conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", + (extracted.uid,)) + conn.execute("UPDATE roles SET seniority = 'lead' WHERE uid = ?", (kept.uid,)) + conn.execute("PRAGMA user_version = 16") + + migrated = Store(path) + + assert migrated.migrations_applied == [17] + with sqlite3.connect(path) as conn: + rows = {uid: (fold, level) for uid, fold, level in conn.execute( + "SELECT uid, place_fold, seniority FROM roles")} + assert rows == {plain.uid: ("kobenhavn k", "senior"), extracted.uid: ("malmo", ""), + kept.uid: ("", "lead"), unknown.uid: ("", "")} + + def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None: ai = Prompt(name="AI leadership", text="Head of AI") fde = Prompt(name="Forward deployed", text="FDE lead") @@ -639,6 +674,53 @@ def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Sto assert role is not None and role.function == "sales" +def _folded(store: Store, uid: str) -> str: + with sqlite3.connect(store.path) as conn: + return str(conn.execute("SELECT place_fold FROM roles WHERE uid = ?", + (uid,)).fetchone()[0]) + + +def test_place_fold_is_written_with_place(store: Store) -> None: + store.upsert_role(posting("1", place="København K")) + uid = posting("1").uid + assert _folded(store, uid) == "kobenhavn k" + store.upsert_role(posting("1")) # says less: both kept + assert (store.role(uid).place, _folded(store, uid)) == ( # type: ignore[union-attr] + "København K", "kobenhavn k") + store.upsert_role(posting("1", place="Kgs. Lyngby")) + assert _folded(store, uid) == "kgs lyngby" + + store.upsert_role(posting("2")) + store.set_extracted(posting("2").uid, {"place": "Malmö, Sweden"}) + assert _folded(store, posting("2").uid) == "malmo sweden" + + +def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> None: + store.upsert_role(posting("1")) # "Senior Backend Engineer" + uid = posting("1").uid + assert store.role(uid).seniority == "senior" # type: ignore[union-attr] + retitled = posting("1") + retitled.title = "Backend Engineer" + store.upsert_role(retitled) # the title no longer says + assert store.role(uid).seniority == "" # type: ignore[union-attr] + + store.upsert_role(posting("1")) + store.set_extracted(uid, {"seniority": "lead"}) # extract overwrites the title's + assert store.role(uid).seniority == "lead" # type: ignore[union-attr] + store.upsert_role(posting("1")) # and a pull does not put it back + assert store.role(uid).seniority == "lead" # type: ignore[union-attr] + store.set_extracted(uid, {}) + assert store.role(uid).seniority == "lead" # type: ignore[union-attr] + + +def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None: + store.upsert_role(posting("1", seniority="junior")) + uid = posting("1").uid + assert store.role(uid).seniority == "junior" # type: ignore[union-attr] + store.set_extracted(uid, {"seniority": "senior"}) + assert store.role(uid).seniority == "junior" # type: ignore[union-attr] + + def test_company_falls_back_to_the_subscription(store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse") store.upsert_role(posting(), sub) @@ -749,6 +831,116 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No store.roles(pay="bogus") +def test_roles_filter_by_company_keys(store: Store) -> None: + store.upsert_role(posting("1", company="Acme")) + store.upsert_role(posting("2", company="Acme Labs")) + store.upsert_role(posting("3", company="Other")) + assert {r.source_id for r in store.roles(company_keys=["acme"])} == {"1"} + assert {r.source_id for r in store.roles(company_keys=["acme", "other"])} == {"1", "3"} + assert store.count_roles(company_keys=["acme-labs"]) == 1 + assert store.roles(company_keys=["nobody"]) == [] + + +def test_roles_filter_by_place_tokens_on_whole_words(store: Store) -> None: + for sid, place, country in ( + ("kbh", "København K", ""), ("ballerup", "Ballerup", ""), + ("offices", "Copenhagen; London; Warsaw", ""), ("aarhus", "Aarhus C", "DK"), + ("malmo", "Malmö", ""), ("lund", "Lund", "SE"), ("billund", "Billund", "DK"), + ("berlin", "Berlin", "")): + store.upsert_role(posting(sid, place=place, country=country)) + + def found(*tokens: str) -> set[str]: + return {r.source_id for r in store.roles(places=tokens)} + + assert found("country:DK") == {"kbh", "ballerup", "offices", "aarhus", "billund"} + assert found("region:skane") == {"malmo", "lund"} + assert found("country:DK", "region:skane") == \ + {"kbh", "ballerup", "offices", "aarhus", "billund", "malmo", "lund"} + assert found("place:lund") == {"lund"} # not Billund + assert found("Lund") == {"lund"} # bare text is a place + assert found("place:copenhagen") == {"kbh", "offices"} # and its aliases + assert store.count_roles(places=["region:skane"]) == 2 + with pytest.raises(ValueError, match="region:atlantis"): + store.roles(places=["region:atlantis"]) + + +def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store) -> None: + for sid, title in (("senior", "Senior Engineer"), ("junior", "Junior Engineer"), + ("lead", "Tech Lead"), ("none", "Engineer")): + item = posting(sid) + item.title = title + store.upsert_role(item) + + def found(*levels: str, unknown: bool = True) -> set[str]: + return {r.source_id for r in store.roles(seniority=levels, seniority_unknown=unknown)} + + assert found() == {"senior", "junior", "lead", "none"} + assert found("senior") == {"senior", "none"} + assert found("senior", "lead") == {"senior", "lead", "none"} + assert found("senior", "lead", unknown=False) == {"senior", "lead"} + assert store.count_roles(seniority=["junior"], seniority_unknown=False) == 1 + assert "intern" in LEVELS and "" not in LEVELS + with pytest.raises(ValueError, match="seniority is"): + store.roles(seniority=["boss"]) + + +def test_roles_pay_unknown_lets_the_roles_with_no_pay_through(store: Store) -> None: + store.upsert_role(posting("normalised", salary_min=70000, currency="EUR")) + store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour")) + store.upsert_role(posting("none")) + assert {r.source_id for r in store.roles(pay="normalised", pay_unknown=True)} == \ + {"normalised", "none"} + assert {r.source_id for r in store.roles(pay="stated", pay_unknown=True)} == \ + {"normalised", "stated", "none"} + assert store.count_roles(pay="normalised") == 1 + assert store.count_roles(pay_unknown=True) == 3 # no pay filter, nothing to widen + + +def test_role_facets_count_each_facet_with_its_own_filter_left_out(store: Store) -> None: + for sid, title, raw in ( + ("a", "Senior Engineer", {"salary_min": 70000, "currency": "EUR"}), + ("b", "Senior Engineer", {"salary_text": "USD 45 per hour"}), + ("c", "Junior Engineer", {}), + ("d", "Engineer", {}), + ("e", "Senior Engineer", {"company": "Other"})): + item = posting(sid, **raw) + item.title = title + store.upsert_role(item) + + facets = store.role_facets(seniority=["junior"], seniority_unknown=False, + pay="normalised") + # Seniority counted under the pay filter only: a. + assert dict(facets["seniority"]) == {**dict.fromkeys(LEVELS, 0), "senior": 1, "": 0} + # Pay counted under the seniority filter only: c. + assert facets["pay"] == [("normalised", 0), ("stated", 0), ("", 1)] + assert [v for v, _ in facets["seniority"]] == [*LEVELS, ""] + + everything = store.role_facets() + assert dict(everything["seniority"])["senior"] == 3 + assert dict(everything["seniority"])[""] == 1 + assert everything["pay"] == [("normalised", 1), ("stated", 1), ("", 3)] + assert dict(store.role_facets(company_keys=["other"])["seniority"])["senior"] == 1 + + +def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store) -> None: + # One key, three spellings: the company row's name first, then the roles' + # by how many carry each. + store.add_company("Ørsted Group", key="ørsted") + store.upsert_role(posting("1", company="Ørsted A/S", place="Gentofte")) + store.upsert_role(posting("2", company="Ørsted", place="Gentofte")) + store.upsert_role(posting("3", company="Ørsted", place="Malmö")) + store.upsert_role(posting("4", company="Hidden", place="Malmö")) + agent = store.save_agent("A") + store.set_affinity(agent.id, "hidden", -1) + store.set_affinity(agent.id, "ørsted", 2) + + spellings = ["Ørsted Group", "Ørsted", "Ørsted A/S"] + assert sorted(store.open_companies()) == [ + ("hidden", ["Hidden"], 1, 0), ("ørsted", spellings, 3, 0)] + assert store.open_companies(agent.id) == [("ørsted", spellings, 3, 2)] + assert sorted(store.open_places()) == [("Gentofte", "", 2), ("Malmö", "SE", 2)] + + def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None: agent = store.save_agent("A") store.add_company("Acme", key="acme") @@ -1116,6 +1308,24 @@ def test_the_default_agent_is_the_lowest_enabled_id(store: Store) -> None: assert store.default_agent() == later +def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None: + for sid, place, country in (("lund", "Lund", "SE"), ("billund", "Billund", "DK"), + ("kbh", "København K", ""), ("none", "", "")): + store.upsert_role(posting(sid, place=place, country=country)) + agent = store.save_agent("A") + uid = {r.source_id: r.uid for r in store.roles()} + + def states(*values: str) -> dict[str, int]: + found = store.score_candidates(agent.id, [Gate(field="place", values=list(values))]) + return {sid: next(c.gates[0] for c in found if c.uid == u) for sid, u in uid.items()} + + assert states("Lund") == {"lund": 1, "billund": 0, "kbh": 0, "none": -1} + assert states("Copenhagen", "region:skane") == {"lund": 1, "billund": 0, "kbh": 1, + "none": -1} + assert states("country:DK") == {"lund": 0, "billund": 1, "kbh": 1, "none": -1} + assert states("region:atlantis") == {"lund": 0, "billund": 0, "kbh": 0, "none": -1} + + def test_open_role_counts_and_company_boards(store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse") store.add_company("Acme", key="acme") diff --git a/tests/test_web.py b/tests/test_web.py index b13b931..bfa4e57 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -91,6 +91,111 @@ def test_list_roles_filters_by_query_place_and_function( assert [r["uid"] for r in res.json()["items"]] == [uid["3"]] +def test_list_roles_filters_by_company_keys_places_and_seniority( + client: TestClient, store: Store) -> None: + for sid, company, place, title in ( + ("kbh", "Acme", "København K", "Senior Engineer"), + ("malmo", "Acme", "Malmö", "Engineer"), + ("billund", "Other", "Billund, Denmark", "Junior Engineer"), + ("berlin", "Other", "Berlin", "Senior Engineer")): + item = posting(sid, company=company, place=place) + item.title = title + store.upsert_role(item) + uid = {r.source_id: r.uid for r in store.roles()} + + def shown(**params: Any) -> set[str]: + res = client.get("/api/roles", params=params) + assert res.status_code == 200, res.text + assert res.json()["total"] == len(res.json()["items"]) + return {sid for sid, u in uid.items() if u in {r["uid"] for r in res.json()["items"]}} + + assert shown(company_keys=["acme"]) == {"kbh", "malmo"} + assert shown(company_keys=["acme", "other"]) == set(uid) + assert shown(places=["country:DK"]) == {"kbh", "billund"} + assert shown(places=["country:DK", "region:skane"]) == {"kbh", "billund", "malmo"} + assert shown(places=["place:kobenhavn"]) == {"kbh"} + assert shown(places=["Lund"]) == set() # not Billund + assert shown(seniority=["senior"]) == {"kbh", "berlin", "malmo"} + assert shown(seniority=["senior"], seniority_unknown="false") == {"kbh", "berlin"} + assert shown(seniority=["senior", "junior"], seniority_unknown="false", + company_keys=["other"]) == {"billund", "berlin"} + + res = client.get("/api/roles", params={"places": ["region:atlantis"]}) + assert res.status_code == 422 and "region:atlantis" in res.json()["detail"] + res = client.get("/api/roles", params={"seniority": ["boss"]}) + assert res.status_code == 422 and "boss" in res.json()["detail"] + + +def test_role_facets_count_each_facet_under_the_other_filters( + client: TestClient, store: Store) -> None: + for sid, title, raw in (("a", "Senior Engineer", {"salary_min": 70000}), + ("b", "Senior Engineer", {}), + ("c", "Engineer", {"salary_text": "EUR 50 per hour"})): + item = posting(sid, **raw) + item.title = title + store.upsert_role(item) + + res = client.get("/api/roles/facets", + params={"seniority": ["senior"], "seniority_unknown": "false", + "pay": "stated"}) + assert res.status_code == 200 + body = res.json() + levels = {f["value"]: f["count"] for f in body["seniority"]} + assert (levels["senior"], levels[""], levels["junior"]) == (1, 1, 0) # a and c pay + assert body["pay"] == [{"value": "normalised", "count": 1}, # a and b are senior + {"value": "stated", "count": 0}, {"value": "", "count": 1}] + assert client.get("/api/roles/facets", params={"places": ["country:ZZ"]}).status_code == 422 + + +def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden( + client: TestClient, store: Store) -> None: + for sid, company in (("1", "Ørsted"), ("2", "Ørsted"), ("3", "Data Minds"), + ("4", "Big Data Corp"), ("5", "Big Data Corp"), ("6", "Big Data Corp"), + ("7", "Metadata Ltd"), ("8", "Hidden Data")): + store.upsert_role(posting(sid, company=company)) + agent = store.save_agent("A") + store.set_affinity(agent.id, "hidden-data", -1) + store.set_affinity(agent.id, "ørsted", 2) + + def suggest(q: str, **params: Any) -> list[dict[str, Any]]: + res = client.get("/api/suggest/companies", params={"q": q, **params}) + assert res.status_code == 200 + return res.json() + + assert suggest("orsted") == [{"key": "ørsted", "name": "Ørsted", "count": 2, + "affinity": 2}] + # A word prefix, not a substring: Metadata is not found; by open roles. + assert [c["name"] for c in suggest("data")] == ["Big Data Corp", "Data Minds"] + assert [c["name"] for c in suggest("big da")] == ["Big Data Corp"] + assert [c["name"] for c in suggest("data", limit=1)] == ["Big Data Corp"] + assert suggest("") == [] + # Another agent hid nothing. + other = store.save_agent("B") + assert "Hidden Data" in [c["name"] for c in suggest("data", agent=other.id)] + + +def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None: + for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"), + ("4", "København K"), ("5", "Kolding")): + store.upsert_role(posting(sid, place=place)) + + res = client.get("/api/suggest/places", params={"q": "ska"}) + assert res.status_code == 200 + assert res.json()[0] == {"token": "region:skane", "label": "Skåne", "kind": "region", + "count": 3} + tokens = [s["token"] for s in client.get("/api/suggest/places", + params={"q": "kob"}).json()] + assert "place:kobenhavn" in tokens + assert client.get("/api/suggest/places", params={"q": ""}).json() == [] + + res = client.get("/api/places/labels", + params={"tokens": ["country:DK", "region:skane", "place:malmo", + "region:atlantis"]}) + assert res.json() == [{"token": "country:DK", "label": "Denmark"}, + {"token": "region:skane", "label": "Skåne"}, + {"token": "place:malmo", "label": "Malmö"}] + + def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store) -> None: """`salary_text` and `salary_state` (app/models.py) are on every role the API serves, and `pay` (`Store.PAY`) filters the feed by it, in SQL before @@ -1052,13 +1157,14 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns( row = client.get("/api/roles").json()["items"][0] assert set(row) == { "uid", "company_key", "company", "title", "place", "country", "remote", "function", - "salary_min", "salary_max", "currency", "salary_text", "salary_state", + "seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state", "original_url", "posted_at", "score", "gated_reason", "prompt", "vote", "voted_at", "read_at", "tags", "excerpt", } assert row["uid"] == uid assert row["company"] == "Acme" and row["place"] == "Berlin, Germany" assert row["remote"] == "hybrid" and row["function"] == "engineering" + assert row["seniority"] == "senior" # "Senior Backend Engineer" assert len(row["excerpt"]) <= 200 assert row["excerpt"] == models_excerpt(long_description)