Keep a board's stated pay as salary_text, even when it does not normalise

Every ats parser now writes one human-readable pay line whether or not it
converts to yearly salary_min/salary_max/currency: hourly and daily rates,
Greenhouse's several pay zones, and a range with no currency all survive as
text instead of being dropped outright. Stored in roles.salary_text
(migration 12), replaced whole on every pull like title, so a board that
stops stating pay clears it rather than leaving a stale line. salary_state
is derived, not stored ('normalised', 'stated', or ''), and is what the
feed's new Pay filter and badge read; gates and scoring are unchanged, still
reading only the normalised columns.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet committed 2026-09-29 15:08:45 +02:00
1 parent 706b127bc0
commit fa4e1a657f
18 files changed
+528 -57

No files matched your search

+19 -1
View File
@@ -15,7 +15,7 @@ import re
import unicodedata
from typing import Annotated, Any, Literal, get_args
from pydantic import BeforeValidator, Field
from pydantic import BeforeValidator, Field, computed_field
from desk.models import JsonDict, Model, ModelTaskResult
@@ -337,6 +337,10 @@ class Role(Model):
salary_min: int | None = None
salary_max: int | None = None
currency: str = ""
#: The pay a board stated, as one human-readable line, whether or not it
#: could be normalised into `salary_min`/`salary_max`/`currency`
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
salary_text: str = ""
seniority: str = ""
function: str = ""
permit: Permit = ""
@@ -372,6 +376,20 @@ class Role(Model):
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
tags: list[str] = []
@computed_field # type: ignore[prop-decorator]
@property
def salary_state(self) -> Literal["normalised", "stated", ""]:
"""Not a column: 'normalised' when `salary_min` or `salary_max` is
set, 'stated' when only `salary_text` is, '' otherwise. Gates and
scoring read `salary_min` alone and never this (docs/decisions.md);
it is what the feed's Pay filter and badge read instead
(`Store.PAY`, `static/roles.js`)."""
if self.salary_min is not None or self.salary_max is not None:
return "normalised"
if self.salary_text:
return "stated"
return ""
class ExtractResult(ModelTaskResult):
"""What one run of `extract` did. See app/jobs/extract.py."""
+44 -10
View File
@@ -84,7 +84,7 @@
function blankFilter() {
return {
sort: "score", agent: null, period: "all", tags: new Set(), showRead: true,
company: "", place: "", function: "", remote: "", judged: "",
company: "", place: "", function: "", remote: "", judged: "", pay: "",
showMuted: false, gated: false,
};
}
@@ -131,7 +131,7 @@
const p = new URLSearchParams();
if (state.q) p.set("q", state.q);
for (const key of ["company", "place"]) if (f[key].trim()) p.set(key, f[key].trim());
for (const key of ["function", "remote", "judged"]) if (f[key]) p.set(key, f[key]);
for (const key of ["function", "remote", "judged", "pay"]) if (f[key]) p.set(key, f[key]);
const days = PERIODS[f.period].days;
if (days) p.set("days", String(days));
if (f.showMuted) p.set("show_muted", "true");
@@ -154,7 +154,7 @@
const q = new URLSearchParams(location.search);
const f = state.filter;
state.q = q.get("q") || "";
for (const key of ["company", "place", "function", "remote", "judged"]) {
for (const key of ["company", "place", "function", "remote", "judged", "pay"]) {
f[key] = q.get(key) || "";
}
f.period = PERIOD_OF_DAYS[q.get("days")] || "all";
@@ -165,7 +165,8 @@
if (q.get("sort") === "time") f.sort = "time";
// Checked against the enabled agents once they are in (`chooseAgent`).
if (q.get("agent")) f.agent = Number(q.get("agent"));
if (f.company || f.place || f.function || f.remote || f.judged || f.showMuted || f.gated) {
if (f.company || f.place || f.function || f.remote || f.judged || f.pay
|| f.showMuted || f.gated) {
$("#f-more").open = true;
}
}
@@ -389,10 +390,11 @@
setValue("f-place", f.place);
setValue("f-function", f.function);
setValue("f-remote", f.remote);
setValue("f-pay", f.pay);
setValue("f-judged", f.judged);
setValue("f-show-muted", f.showMuted);
setValue("f-gated", f.gated);
const more = [f.company.trim(), f.place.trim(), f.function, f.remote, f.judged,
const more = [f.company.trim(), f.place.trim(), f.function, f.remote, f.pay, f.judged,
f.showMuted, ranked && f.gated].filter(Boolean).length;
$("#f-active").textContent = more ? ` · ${more}` : "";
@@ -435,7 +437,7 @@
function renderActive() {
const f = state.filter;
const on = [!f.showRead, f.period !== "all", f.company.trim(), f.place.trim(),
f.function, f.remote, f.judged, f.showMuted, sortShown() === "score" && f.gated]
f.function, f.remote, f.pay, f.judged, f.showMuted, sortShown() === "score" && f.gated]
.filter(Boolean).length + f.tags.size;
const count = on === 1 ? "1 filter" : `${on} filters`;
$(".rail-toggle-label").textContent = on ? count : "Menu";
@@ -490,9 +492,20 @@
aria-pressed="${state.filter.tags.has(tag)}">${esc(tag)}</button>`).join("");
}
/** The feed row's small pay badge: the range for 'normalised', "pay
stated" for 'stated', nothing for '' — `role.salary_state`, computed by
the API from `salary_min`/`salary_max`/`salary_text` (app/models.py). */
function payBadge(role) {
if (role.salary_state === "normalised") return salary(role);
if (role.salary_state === "stated") return "pay stated";
return "";
}
function badges(role, { seniority = false } = {}) {
const words = [role.remote, role.function, seniority ? role.seniority : ""].filter(Boolean);
const pay = payBadge(role);
return (role.closed_at ? `<span class="badge closed">closed</span>` : "")
+ (pay ? `<span class="badge">${esc(pay)}</span>` : "")
+ words.map((w) => `<span class="badge">${esc(w)}</span>`).join("");
}
@@ -575,7 +588,7 @@
if (!empty.hidden) {
const f = state.filter;
const filtered = state.q || f.tags.size || f.period !== "all" || !f.showRead
|| f.company.trim() || f.place.trim() || f.function || f.remote || f.judged;
|| f.company.trim() || f.place.trim() || f.function || f.remote || f.pay || f.judged;
empty.innerHTML = filtered
? `<h2>Nothing matches</h2>
<p>No open role fits these filters.
@@ -701,6 +714,25 @@
return "";
}
/* What the reader shows below the title for pay, or null when the role
states none: 'normalised' shows the range from the normalised columns,
plus the board's own text underneath when it says more than the range
does — a period, several zones — spotted by " per " or ";" in it, which
every conforming `salary_text` carries and the plain range never does
(`plugins.ats`'s `_pay_line`); 'stated' shows the text itself, with a
note that it was not normalised. */
function payDisplay(r) {
if (r.salary_state === "normalised") {
const range = salary(r);
const text = r.salary_text || "";
return { range, detail: /(;| per )/.test(text) && text !== range ? text : "", note: "" };
}
if (r.salary_state === "stated") {
return { range: r.salary_text, detail: "", note: "as stated, not normalised" };
}
return null;
}
function host(url) {
try { return new URL(url).host.replace(/^www\./, ""); } catch (e) { return ""; }
}
@@ -740,7 +772,7 @@
? `<p class="reader-note">Loading…</p>`
: text ? paragraphs(text)
: `<p class="reader-note">The board gave no description for this one.</p>`;
const pay = salary(r);
const pay = payDisplay(r);
const waiting = state.summarising === state.readingUid;
const teaser = r.uid && !r.article_text && out && (r.description || "").length < TEASER;
@@ -763,7 +795,9 @@
${tagChips(r.tags)}
</div>
<h1>${esc(r.title || "")}</h1>
${pay ? `<p class="reader-salary">${esc(pay)}</p>` : ""}
${pay ? `<p class="reader-salary">${esc(pay.range)}</p>
${pay.detail ? `<p class="muted small">${esc(pay.detail)}</p>` : ""}
${pay.note ? `<p class="muted small">${esc(pay.note)}</p>` : ""}` : ""}
${r.summary ? `<section class="reader-summary">
<h2 class="reader-label">Summary</h2><p>${esc(r.summary)}</p></section>` : ""}
<div class="reader-tools">
@@ -1030,7 +1064,7 @@
for (const key of ["company", "place"]) {
$(`#f-${key}`).addEventListener("input", (e) => { state.filter[key] = e.target.value; typed(); });
}
for (const key of ["function", "remote", "judged"]) {
for (const key of ["function", "remote", "pay", "judged"]) {
$(`#f-${key}`).addEventListener("change", (e) => setFilter({ [key]: e.target.value }));
}
$("#f-show-muted").addEventListener("change",
+6 -1
View File
@@ -86,7 +86,8 @@ CREATE TABLE IF NOT EXISTS roles (
-- whose description is only a teaser.
article_text TEXT NOT NULL DEFAULT '',
article_at TEXT,
created_at TEXT NOT NULL
created_at TEXT NOT NULL,
salary_text TEXT NOT NULL DEFAULT ''
);
-- The open vocabulary summarise reads off a posting: the role family, the
@@ -381,6 +382,10 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
# its current rules is the whole point (see _recompute_function below).
(11, "roles.function recomputed for every not-yet-extracted role, word-boundary "
"classifier fix", lambda conn: _recompute_function(conn)),
(12, "roles.salary_text: the pay a board stated as one line, kept even when it "
"could not be normalised into salary_min/salary_max/currency", """
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
"""),
)
#: What a file created from `SCHEMA` today is already at.
+31 -7
View File
@@ -60,6 +60,17 @@ JUDGED: dict[str, str] = {
"passed": "v.vote = -1",
}
#: The feed's Pay filter, `_roles_from`'s `pay`: the SQL that is true for
#: each choice, over `roles`. `salary_state` (`app.models.Role`) is derived,
#: not stored, so the filter spells the same rule out in SQL rather than
#: reading a column: 'normalised' is `salary_min` or `salary_max` set,
#: 'stated' is `salary_text` set and neither of those is.
PAY: dict[str, str] = {
"stated": "(salary_min IS NOT NULL OR salary_max IS NOT NULL OR salary_text != '')",
"normalised": "(salary_min IS NOT NULL OR salary_max IS NOT NULL)",
"stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)",
}
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
#: moves the role at once and the stored score stays what `score` computed.
@@ -313,6 +324,10 @@ class Store:
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
# must be able to clear it, not leave a stale line behind.
row["salary_text"] = str(raw.get("salary_text") or "")
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
@@ -320,7 +335,8 @@ class Store:
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -457,13 +473,15 @@ class Store:
hide_muted: bool = False, include_read: bool = True,
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
judged: str | None = None, pay: str | None = None,
offset: int = 0) -> list[ScoredRole]:
"""Newest first by `posted_at`. `company` and `place` are
case-insensitive substrings of the columns a board wrote — not the
key, so a search does not have to know the exact slug a company folds
to. `q` is matched word by word against `title`; a role must match
every word. `function` and `remote` match a gate column exactly.
`days` keeps only roles posted within that many days of now.
`days` keeps only roles posted within that many days of now. `pay`
(a key of `PAY`) filters on `salary_state`.
`agent` says whose votes, hidden companies and scores are meant. Each
row carries that agent's `vote` and `voted_at`; `judged` (a key of
@@ -498,7 +516,7 @@ class Store:
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
judged=judged, pay=pay)
select = "SELECT roles.*, v.vote, v.voted_at"
order = "roles.posted_at DESC, roles.uid"
scored = agent is not None and by_score
@@ -527,13 +545,14 @@ class Store:
days: int | None = None, hide_muted: bool = False,
include_read: bool = True, tags: Sequence[str] = (),
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
gated: bool = False, judged: str | None = None) -> int:
gated: bool = False, judged: str | None = None,
pay: str | None = None) -> int:
"""How many roles `roles` would list with the same filters and no limit."""
sql, params = self._roles_from(
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
judged=judged, pay=pay)
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
return int(row["n"]) if row else 0
@@ -541,7 +560,8 @@ class Store:
q: str | None, function: str | None, remote: str | None,
days: int | None, hide_muted: bool, include_read: bool,
tags: Sequence[str], tag_mode: str, agent: int | None,
by_score: bool, gated: bool, judged: str | None) -> tuple[str, list[Any]]:
by_score: bool, gated: bool, judged: str | None,
pay: str | None = None) -> tuple[str, list[Any]]:
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
its parameters in order. The agent's votes are always joined as `v`
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
@@ -569,6 +589,10 @@ class Store:
if judged not in JUDGED:
raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}")
where.append(JUDGED[judged])
if pay:
if pay not in PAY:
raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}")
where.append(PAY[pay])
if open_only:
where.append("closed_at IS NULL")
if hide_muted and agent is not None:
+6
View File
@@ -84,6 +84,12 @@
<option value="{{ value }}">{{ label }}</option>
{% endfor %}
</select>
<select id="f-pay" aria-label="Pay">
<option value="">Any pay</option>
{% for value, label in pay_options %}
<option value="{{ value }}">{{ label }}</option>
{% endfor %}
</select>
<select id="f-judged" aria-label="Judged">
<option value="">Any vote</option>
<option value="liked">Liked</option>
+5 -2
View File
@@ -134,6 +134,7 @@ def list_roles(
order: Literal["score", "newest"] = "score",
gated: bool = False,
judged: Literal["", "liked", "unjudged", "passed"] = "",
pay: Literal["", "stated", "normalised", "stated_only"] = "",
offset: Annotated[int, Query(ge=0)] = 0,
limit: Annotated[int, Query(ge=1, le=1000)] = 60,
) -> RolesPage:
@@ -152,7 +153,9 @@ def list_roles(
`include_read` set false hides a role already marked read. `tags` is
`?tags=a&tags=b`, kept to a role with any of them or, with
`tag_mode=all`, every one."""
`tag_mode=all`, every one. `pay` keeps only roles whose `salary_state`
is stated or normalised, normalised only, or stated but not normalised
(`Store.PAY`)."""
agent_id = _agent_id(store, agent)
by_score = agent is not None and order == "score"
filters: dict[str, Any] = {
@@ -160,7 +163,7 @@ def list_roles(
"function": function or None, "remote": remote or None, "days": days,
"hide_muted": not show_muted, "include_read": include_read, "tags": tags,
"tag_mode": tag_mode, "agent": agent_id, "by_score": by_score,
"gated": gated, "judged": judged or None}
"gated": gated, "judged": judged or None, "pay": pay or None}
return RolesPage(items=store.roles(limit=limit, offset=offset, **filters),
total=store.count_roles(**filters))
+7 -2
View File
@@ -116,6 +116,10 @@ MOVED: dict[str, str] = {
#: role's `function` column holds; only the label is dressed up.
FUNCTION_OPTIONS = [(f, f.replace("/", " / ").capitalize()) for f in FUNCTIONS]
REMOTE_OPTIONS = [(r, r.capitalize()) for r in REMOTE if r]
#: The feed's Pay filter, in `Store.PAY`'s own keys — see its docstring for
#: what each one keeps.
PAY_OPTIONS = [("stated", "Stated or normalised"), ("normalised", "Normalised only"),
("stated_only", "Stated but not normalised")]
STATUS_OPTIONS = [(s, s.capitalize()) for s in STATUSES]
AREA_OPTIONS = list(JOBINDEX_AREAS.items())
COUNTRY_OPTIONS = [(c, c.upper()) for c in sorted(ADZUNA_COUNTRIES)]
@@ -151,7 +155,7 @@ def page(request: Request, name: str, **context: object) -> HTMLResponse:
@router.get("/", response_class=HTMLResponse)
def roles(request: Request) -> HTMLResponse:
return page(request, "roles.html", functions=FUNCTION_OPTIONS,
remote_options=REMOTE_OPTIONS, role=None)
remote_options=REMOTE_OPTIONS, pay_options=PAY_OPTIONS, role=None)
@router.get("/roles/{uid}", response_class=HTMLResponse)
@@ -165,7 +169,8 @@ def role(request: Request, uid: str, store: StoreDep) -> HTMLResponse:
if found is None:
raise HTTPException(404, f"no such role: {uid}")
return page(request, "roles.html", functions=FUNCTION_OPTIONS,
remote_options=REMOTE_OPTIONS, role=found, active_path="/")
remote_options=REMOTE_OPTIONS, pay_options=PAY_OPTIONS,
role=found, active_path="/")
@router.get("/applications", response_class=HTMLResponse)