Keep a board's stated pay as salary_text, even when it does not normalise
Every ats parser now writes one human-readable pay line whether or not it
converts to yearly salary_min/salary_max/currency: hourly and daily rates,
Greenhouse's several pay zones, and a range with no currency all survive as
text instead of being dropped outright. Stored in roles.salary_text
(migration 12), replaced whole on every pull like title, so a board that
stops stating pay clears it rather than leaving a stale line. salary_state
is derived, not stored ('normalised', 'stated', or ''), and is what the
feed's new Pay filter and badge read; gates and scoring are unchanged, still
reading only the normalised columns.
Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
fa4e1a657f
18 files changed
+528
-57
No files matched your search
+19
-1
@@ -15,7 +15,7 @@ import re
|
||||
import unicodedata
|
||||
from typing import Annotated, Any, Literal, get_args
|
||||
|
||||
from pydantic import BeforeValidator, Field
|
||||
from pydantic import BeforeValidator, Field, computed_field
|
||||
|
||||
from desk.models import JsonDict, Model, ModelTaskResult
|
||||
|
||||
@@ -337,6 +337,10 @@ class Role(Model):
|
||||
salary_min: int | None = None
|
||||
salary_max: int | None = None
|
||||
currency: str = ""
|
||||
#: The pay a board stated, as one human-readable line, whether or not it
|
||||
#: could be normalised into `salary_min`/`salary_max`/`currency`
|
||||
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
|
||||
salary_text: str = ""
|
||||
seniority: str = ""
|
||||
function: str = ""
|
||||
permit: Permit = ""
|
||||
@@ -372,6 +376,20 @@ class Role(Model):
|
||||
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
|
||||
tags: list[str] = []
|
||||
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
def salary_state(self) -> Literal["normalised", "stated", ""]:
|
||||
"""Not a column: 'normalised' when `salary_min` or `salary_max` is
|
||||
set, 'stated' when only `salary_text` is, '' otherwise. Gates and
|
||||
scoring read `salary_min` alone and never this (docs/decisions.md);
|
||||
it is what the feed's Pay filter and badge read instead
|
||||
(`Store.PAY`, `static/roles.js`)."""
|
||||
if self.salary_min is not None or self.salary_max is not None:
|
||||
return "normalised"
|
||||
if self.salary_text:
|
||||
return "stated"
|
||||
return ""
|
||||
|
||||
|
||||
class ExtractResult(ModelTaskResult):
|
||||
"""What one run of `extract` did. See app/jobs/extract.py."""
|
||||
|
||||
+44
-10
@@ -84,7 +84,7 @@
|
||||
function blankFilter() {
|
||||
return {
|
||||
sort: "score", agent: null, period: "all", tags: new Set(), showRead: true,
|
||||
company: "", place: "", function: "", remote: "", judged: "",
|
||||
company: "", place: "", function: "", remote: "", judged: "", pay: "",
|
||||
showMuted: false, gated: false,
|
||||
};
|
||||
}
|
||||
@@ -131,7 +131,7 @@
|
||||
const p = new URLSearchParams();
|
||||
if (state.q) p.set("q", state.q);
|
||||
for (const key of ["company", "place"]) if (f[key].trim()) p.set(key, f[key].trim());
|
||||
for (const key of ["function", "remote", "judged"]) if (f[key]) p.set(key, f[key]);
|
||||
for (const key of ["function", "remote", "judged", "pay"]) if (f[key]) p.set(key, f[key]);
|
||||
const days = PERIODS[f.period].days;
|
||||
if (days) p.set("days", String(days));
|
||||
if (f.showMuted) p.set("show_muted", "true");
|
||||
@@ -154,7 +154,7 @@
|
||||
const q = new URLSearchParams(location.search);
|
||||
const f = state.filter;
|
||||
state.q = q.get("q") || "";
|
||||
for (const key of ["company", "place", "function", "remote", "judged"]) {
|
||||
for (const key of ["company", "place", "function", "remote", "judged", "pay"]) {
|
||||
f[key] = q.get(key) || "";
|
||||
}
|
||||
f.period = PERIOD_OF_DAYS[q.get("days")] || "all";
|
||||
@@ -165,7 +165,8 @@
|
||||
if (q.get("sort") === "time") f.sort = "time";
|
||||
// Checked against the enabled agents once they are in (`chooseAgent`).
|
||||
if (q.get("agent")) f.agent = Number(q.get("agent"));
|
||||
if (f.company || f.place || f.function || f.remote || f.judged || f.showMuted || f.gated) {
|
||||
if (f.company || f.place || f.function || f.remote || f.judged || f.pay
|
||||
|| f.showMuted || f.gated) {
|
||||
$("#f-more").open = true;
|
||||
}
|
||||
}
|
||||
@@ -389,10 +390,11 @@
|
||||
setValue("f-place", f.place);
|
||||
setValue("f-function", f.function);
|
||||
setValue("f-remote", f.remote);
|
||||
setValue("f-pay", f.pay);
|
||||
setValue("f-judged", f.judged);
|
||||
setValue("f-show-muted", f.showMuted);
|
||||
setValue("f-gated", f.gated);
|
||||
const more = [f.company.trim(), f.place.trim(), f.function, f.remote, f.judged,
|
||||
const more = [f.company.trim(), f.place.trim(), f.function, f.remote, f.pay, f.judged,
|
||||
f.showMuted, ranked && f.gated].filter(Boolean).length;
|
||||
$("#f-active").textContent = more ? ` · ${more}` : "";
|
||||
|
||||
@@ -435,7 +437,7 @@
|
||||
function renderActive() {
|
||||
const f = state.filter;
|
||||
const on = [!f.showRead, f.period !== "all", f.company.trim(), f.place.trim(),
|
||||
f.function, f.remote, f.judged, f.showMuted, sortShown() === "score" && f.gated]
|
||||
f.function, f.remote, f.pay, f.judged, f.showMuted, sortShown() === "score" && f.gated]
|
||||
.filter(Boolean).length + f.tags.size;
|
||||
const count = on === 1 ? "1 filter" : `${on} filters`;
|
||||
$(".rail-toggle-label").textContent = on ? count : "Menu";
|
||||
@@ -490,9 +492,20 @@
|
||||
aria-pressed="${state.filter.tags.has(tag)}">${esc(tag)}</button>`).join("");
|
||||
}
|
||||
|
||||
/** The feed row's small pay badge: the range for 'normalised', "pay
|
||||
stated" for 'stated', nothing for '' — `role.salary_state`, computed by
|
||||
the API from `salary_min`/`salary_max`/`salary_text` (app/models.py). */
|
||||
function payBadge(role) {
|
||||
if (role.salary_state === "normalised") return salary(role);
|
||||
if (role.salary_state === "stated") return "pay stated";
|
||||
return "";
|
||||
}
|
||||
|
||||
function badges(role, { seniority = false } = {}) {
|
||||
const words = [role.remote, role.function, seniority ? role.seniority : ""].filter(Boolean);
|
||||
const pay = payBadge(role);
|
||||
return (role.closed_at ? `<span class="badge closed">closed</span>` : "")
|
||||
+ (pay ? `<span class="badge">${esc(pay)}</span>` : "")
|
||||
+ words.map((w) => `<span class="badge">${esc(w)}</span>`).join("");
|
||||
}
|
||||
|
||||
@@ -575,7 +588,7 @@
|
||||
if (!empty.hidden) {
|
||||
const f = state.filter;
|
||||
const filtered = state.q || f.tags.size || f.period !== "all" || !f.showRead
|
||||
|| f.company.trim() || f.place.trim() || f.function || f.remote || f.judged;
|
||||
|| f.company.trim() || f.place.trim() || f.function || f.remote || f.pay || f.judged;
|
||||
empty.innerHTML = filtered
|
||||
? `<h2>Nothing matches</h2>
|
||||
<p>No open role fits these filters.
|
||||
@@ -701,6 +714,25 @@
|
||||
return "";
|
||||
}
|
||||
|
||||
/* What the reader shows below the title for pay, or null when the role
|
||||
states none: 'normalised' shows the range from the normalised columns,
|
||||
plus the board's own text underneath when it says more than the range
|
||||
does — a period, several zones — spotted by " per " or ";" in it, which
|
||||
every conforming `salary_text` carries and the plain range never does
|
||||
(`plugins.ats`'s `_pay_line`); 'stated' shows the text itself, with a
|
||||
note that it was not normalised. */
|
||||
function payDisplay(r) {
|
||||
if (r.salary_state === "normalised") {
|
||||
const range = salary(r);
|
||||
const text = r.salary_text || "";
|
||||
return { range, detail: /(;| per )/.test(text) && text !== range ? text : "", note: "" };
|
||||
}
|
||||
if (r.salary_state === "stated") {
|
||||
return { range: r.salary_text, detail: "", note: "as stated, not normalised" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function host(url) {
|
||||
try { return new URL(url).host.replace(/^www\./, ""); } catch (e) { return ""; }
|
||||
}
|
||||
@@ -740,7 +772,7 @@
|
||||
? `<p class="reader-note">Loading…</p>`
|
||||
: text ? paragraphs(text)
|
||||
: `<p class="reader-note">The board gave no description for this one.</p>`;
|
||||
const pay = salary(r);
|
||||
const pay = payDisplay(r);
|
||||
const waiting = state.summarising === state.readingUid;
|
||||
const teaser = r.uid && !r.article_text && out && (r.description || "").length < TEASER;
|
||||
|
||||
@@ -763,7 +795,9 @@
|
||||
${tagChips(r.tags)}
|
||||
</div>
|
||||
<h1>${esc(r.title || "")}</h1>
|
||||
${pay ? `<p class="reader-salary">${esc(pay)}</p>` : ""}
|
||||
${pay ? `<p class="reader-salary">${esc(pay.range)}</p>
|
||||
${pay.detail ? `<p class="muted small">${esc(pay.detail)}</p>` : ""}
|
||||
${pay.note ? `<p class="muted small">${esc(pay.note)}</p>` : ""}` : ""}
|
||||
${r.summary ? `<section class="reader-summary">
|
||||
<h2 class="reader-label">Summary</h2><p>${esc(r.summary)}</p></section>` : ""}
|
||||
<div class="reader-tools">
|
||||
@@ -1030,7 +1064,7 @@
|
||||
for (const key of ["company", "place"]) {
|
||||
$(`#f-${key}`).addEventListener("input", (e) => { state.filter[key] = e.target.value; typed(); });
|
||||
}
|
||||
for (const key of ["function", "remote", "judged"]) {
|
||||
for (const key of ["function", "remote", "pay", "judged"]) {
|
||||
$(`#f-${key}`).addEventListener("change", (e) => setFilter({ [key]: e.target.value }));
|
||||
}
|
||||
$("#f-show-muted").addEventListener("change",
|
||||
|
||||
@@ -86,7 +86,8 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- whose description is only a teaser.
|
||||
article_text TEXT NOT NULL DEFAULT '',
|
||||
article_at TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
created_at TEXT NOT NULL,
|
||||
salary_text TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- The open vocabulary summarise reads off a posting: the role family, the
|
||||
@@ -381,6 +382,10 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
||||
# its current rules is the whole point (see _recompute_function below).
|
||||
(11, "roles.function recomputed for every not-yet-extracted role, word-boundary "
|
||||
"classifier fix", lambda conn: _recompute_function(conn)),
|
||||
(12, "roles.salary_text: the pay a board stated as one line, kept even when it "
|
||||
"could not be normalised into salary_min/salary_max/currency", """
|
||||
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
|
||||
+31
-7
@@ -60,6 +60,17 @@ JUDGED: dict[str, str] = {
|
||||
"passed": "v.vote = -1",
|
||||
}
|
||||
|
||||
#: The feed's Pay filter, `_roles_from`'s `pay`: the SQL that is true for
|
||||
#: each choice, over `roles`. `salary_state` (`app.models.Role`) is derived,
|
||||
#: not stored, so the filter spells the same rule out in SQL rather than
|
||||
#: reading a column: 'normalised' is `salary_min` or `salary_max` set,
|
||||
#: 'stated' is `salary_text` set and neither of those is.
|
||||
PAY: dict[str, str] = {
|
||||
"stated": "(salary_min IS NOT NULL OR salary_max IS NOT NULL OR salary_text != '')",
|
||||
"normalised": "(salary_min IS NOT NULL OR salary_max IS NOT NULL)",
|
||||
"stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)",
|
||||
}
|
||||
|
||||
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
|
||||
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
|
||||
#: moves the role at once and the stored score stays what `score` computed.
|
||||
@@ -313,6 +324,10 @@ class Store:
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
# A board's own pay line, one per pull like title or published_at —
|
||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||
# must be able to clear it, not leave a stale line behind.
|
||||
row["salary_text"] = str(raw.get("salary_text") or "")
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", *_RAW_TEXT)
|
||||
@@ -320,7 +335,8 @@ class Store:
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
||||
"salary_text")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
@@ -457,13 +473,15 @@ class Store:
|
||||
hide_muted: bool = False, include_read: bool = True,
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
|
||||
judged: str | None = None, pay: str | None = None,
|
||||
offset: int = 0) -> list[ScoredRole]:
|
||||
"""Newest first by `posted_at`. `company` and `place` are
|
||||
case-insensitive substrings of the columns a board wrote — not the
|
||||
key, so a search does not have to know the exact slug a company folds
|
||||
to. `q` is matched word by word against `title`; a role must match
|
||||
every word. `function` and `remote` match a gate column exactly.
|
||||
`days` keeps only roles posted within that many days of now.
|
||||
`days` keeps only roles posted within that many days of now. `pay`
|
||||
(a key of `PAY`) filters on `salary_state`.
|
||||
|
||||
`agent` says whose votes, hidden companies and scores are meant. Each
|
||||
row carries that agent's `vote` and `voted_at`; `judged` (a key of
|
||||
@@ -498,7 +516,7 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged)
|
||||
judged=judged, pay=pay)
|
||||
select = "SELECT roles.*, v.vote, v.voted_at"
|
||||
order = "roles.posted_at DESC, roles.uid"
|
||||
scored = agent is not None and by_score
|
||||
@@ -527,13 +545,14 @@ class Store:
|
||||
days: int | None = None, hide_muted: bool = False,
|
||||
include_read: bool = True, tags: Sequence[str] = (),
|
||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||
gated: bool = False, judged: str | None = None) -> int:
|
||||
gated: bool = False, judged: str | None = None,
|
||||
pay: str | None = None) -> int:
|
||||
"""How many roles `roles` would list with the same filters and no limit."""
|
||||
sql, params = self._roles_from(
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged)
|
||||
judged=judged, pay=pay)
|
||||
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
@@ -541,7 +560,8 @@ class Store:
|
||||
q: str | None, function: str | None, remote: str | None,
|
||||
days: int | None, hide_muted: bool, include_read: bool,
|
||||
tags: Sequence[str], tag_mode: str, agent: int | None,
|
||||
by_score: bool, gated: bool, judged: str | None) -> tuple[str, list[Any]]:
|
||||
by_score: bool, gated: bool, judged: str | None,
|
||||
pay: str | None = None) -> tuple[str, list[Any]]:
|
||||
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
||||
its parameters in order. The agent's votes are always joined as `v`
|
||||
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
|
||||
@@ -569,6 +589,10 @@ class Store:
|
||||
if judged not in JUDGED:
|
||||
raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}")
|
||||
where.append(JUDGED[judged])
|
||||
if pay:
|
||||
if pay not in PAY:
|
||||
raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}")
|
||||
where.append(PAY[pay])
|
||||
if open_only:
|
||||
where.append("closed_at IS NULL")
|
||||
if hide_muted and agent is not None:
|
||||
|
||||
@@ -84,6 +84,12 @@
|
||||
<option value="{{ value }}">{{ label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<select id="f-pay" aria-label="Pay">
|
||||
<option value="">Any pay</option>
|
||||
{% for value, label in pay_options %}
|
||||
<option value="{{ value }}">{{ label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<select id="f-judged" aria-label="Judged">
|
||||
<option value="">Any vote</option>
|
||||
<option value="liked">Liked</option>
|
||||
|
||||
+5
-2
@@ -134,6 +134,7 @@ def list_roles(
|
||||
order: Literal["score", "newest"] = "score",
|
||||
gated: bool = False,
|
||||
judged: Literal["", "liked", "unjudged", "passed"] = "",
|
||||
pay: Literal["", "stated", "normalised", "stated_only"] = "",
|
||||
offset: Annotated[int, Query(ge=0)] = 0,
|
||||
limit: Annotated[int, Query(ge=1, le=1000)] = 60,
|
||||
) -> RolesPage:
|
||||
@@ -152,7 +153,9 @@ def list_roles(
|
||||
|
||||
`include_read` set false hides a role already marked read. `tags` is
|
||||
`?tags=a&tags=b`, kept to a role with any of them or, with
|
||||
`tag_mode=all`, every one."""
|
||||
`tag_mode=all`, every one. `pay` keeps only roles whose `salary_state`
|
||||
is stated or normalised, normalised only, or stated but not normalised
|
||||
(`Store.PAY`)."""
|
||||
agent_id = _agent_id(store, agent)
|
||||
by_score = agent is not None and order == "score"
|
||||
filters: dict[str, Any] = {
|
||||
@@ -160,7 +163,7 @@ def list_roles(
|
||||
"function": function or None, "remote": remote or None, "days": days,
|
||||
"hide_muted": not show_muted, "include_read": include_read, "tags": tags,
|
||||
"tag_mode": tag_mode, "agent": agent_id, "by_score": by_score,
|
||||
"gated": gated, "judged": judged or None}
|
||||
"gated": gated, "judged": judged or None, "pay": pay or None}
|
||||
return RolesPage(items=store.roles(limit=limit, offset=offset, **filters),
|
||||
total=store.count_roles(**filters))
|
||||
|
||||
|
||||
+7
-2
@@ -116,6 +116,10 @@ MOVED: dict[str, str] = {
|
||||
#: role's `function` column holds; only the label is dressed up.
|
||||
FUNCTION_OPTIONS = [(f, f.replace("/", " / ").capitalize()) for f in FUNCTIONS]
|
||||
REMOTE_OPTIONS = [(r, r.capitalize()) for r in REMOTE if r]
|
||||
#: The feed's Pay filter, in `Store.PAY`'s own keys — see its docstring for
|
||||
#: what each one keeps.
|
||||
PAY_OPTIONS = [("stated", "Stated or normalised"), ("normalised", "Normalised only"),
|
||||
("stated_only", "Stated but not normalised")]
|
||||
STATUS_OPTIONS = [(s, s.capitalize()) for s in STATUSES]
|
||||
AREA_OPTIONS = list(JOBINDEX_AREAS.items())
|
||||
COUNTRY_OPTIONS = [(c, c.upper()) for c in sorted(ADZUNA_COUNTRIES)]
|
||||
@@ -151,7 +155,7 @@ def page(request: Request, name: str, **context: object) -> HTMLResponse:
|
||||
@router.get("/", response_class=HTMLResponse)
|
||||
def roles(request: Request) -> HTMLResponse:
|
||||
return page(request, "roles.html", functions=FUNCTION_OPTIONS,
|
||||
remote_options=REMOTE_OPTIONS, role=None)
|
||||
remote_options=REMOTE_OPTIONS, pay_options=PAY_OPTIONS, role=None)
|
||||
|
||||
|
||||
@router.get("/roles/{uid}", response_class=HTMLResponse)
|
||||
@@ -165,7 +169,8 @@ def role(request: Request, uid: str, store: StoreDep) -> HTMLResponse:
|
||||
if found is None:
|
||||
raise HTTPException(404, f"no such role: {uid}")
|
||||
return page(request, "roles.html", functions=FUNCTION_OPTIONS,
|
||||
remote_options=REMOTE_OPTIONS, role=found, active_path="/")
|
||||
remote_options=REMOTE_OPTIONS, pay_options=PAY_OPTIONS,
|
||||
role=found, active_path="/")
|
||||
|
||||
|
||||
@router.get("/applications", response_class=HTMLResponse)
|
||||
|
||||
Reference in new issue
Block a user