Keep a board's stated pay as salary_text, even when it does not normalise

Every ats parser now writes one human-readable pay line whether or not it
converts to yearly salary_min/salary_max/currency: hourly and daily rates,
Greenhouse's several pay zones, and a range with no currency all survive as
text instead of being dropped outright. Stored in roles.salary_text
(migration 12), replaced whole on every pull like title, so a board that
stops stating pay clears it rather than leaving a stale line. salary_state
is derived, not stored ('normalised', 'stated', or ''), and is what the
feed's new Pay filter and badge read; gates and scoring are unchanged, still
reading only the normalised columns.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet committed 2026-09-29 15:08:45 +02:00
1 parent 706b127bc0
commit fa4e1a657f
18 files changed
+528 -57

No files matched your search

+6 -1
View File
@@ -86,7 +86,8 @@ CREATE TABLE IF NOT EXISTS roles (
-- whose description is only a teaser.
article_text TEXT NOT NULL DEFAULT '',
article_at TEXT,
created_at TEXT NOT NULL
created_at TEXT NOT NULL,
salary_text TEXT NOT NULL DEFAULT ''
);
-- The open vocabulary summarise reads off a posting: the role family, the
@@ -381,6 +382,10 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
# its current rules is the whole point (see _recompute_function below).
(11, "roles.function recomputed for every not-yet-extracted role, word-boundary "
"classifier fix", lambda conn: _recompute_function(conn)),
(12, "roles.salary_text: the pay a board stated as one line, kept even when it "
"could not be normalised into salary_min/salary_max/currency", """
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
"""),
)
#: What a file created from `SCHEMA` today is already at.
+31 -7
View File
@@ -60,6 +60,17 @@ JUDGED: dict[str, str] = {
"passed": "v.vote = -1",
}
#: The feed's Pay filter, `_roles_from`'s `pay`: the SQL that is true for
#: each choice, over `roles`. `salary_state` (`app.models.Role`) is derived,
#: not stored, so the filter spells the same rule out in SQL rather than
#: reading a column: 'normalised' is `salary_min` or `salary_max` set,
#: 'stated' is `salary_text` set and neither of those is.
PAY: dict[str, str] = {
"stated": "(salary_min IS NOT NULL OR salary_max IS NOT NULL OR salary_text != '')",
"normalised": "(salary_min IS NOT NULL OR salary_max IS NOT NULL)",
"stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)",
}
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
#: moves the role at once and the stored score stays what `score` computed.
@@ -313,6 +324,10 @@ class Store:
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
# must be able to clear it, not leave a stale line behind.
row["salary_text"] = str(raw.get("salary_text") or "")
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
@@ -320,7 +335,8 @@ class Store:
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -457,13 +473,15 @@ class Store:
hide_muted: bool = False, include_read: bool = True,
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
judged: str | None = None, pay: str | None = None,
offset: int = 0) -> list[ScoredRole]:
"""Newest first by `posted_at`. `company` and `place` are
case-insensitive substrings of the columns a board wrote — not the
key, so a search does not have to know the exact slug a company folds
to. `q` is matched word by word against `title`; a role must match
every word. `function` and `remote` match a gate column exactly.
`days` keeps only roles posted within that many days of now.
`days` keeps only roles posted within that many days of now. `pay`
(a key of `PAY`) filters on `salary_state`.
`agent` says whose votes, hidden companies and scores are meant. Each
row carries that agent's `vote` and `voted_at`; `judged` (a key of
@@ -498,7 +516,7 @@ class Store:
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
judged=judged, pay=pay)
select = "SELECT roles.*, v.vote, v.voted_at"
order = "roles.posted_at DESC, roles.uid"
scored = agent is not None and by_score
@@ -527,13 +545,14 @@ class Store:
days: int | None = None, hide_muted: bool = False,
include_read: bool = True, tags: Sequence[str] = (),
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
gated: bool = False, judged: str | None = None) -> int:
gated: bool = False, judged: str | None = None,
pay: str | None = None) -> int:
"""How many roles `roles` would list with the same filters and no limit."""
sql, params = self._roles_from(
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
judged=judged, pay=pay)
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
return int(row["n"]) if row else 0
@@ -541,7 +560,8 @@ class Store:
q: str | None, function: str | None, remote: str | None,
days: int | None, hide_muted: bool, include_read: bool,
tags: Sequence[str], tag_mode: str, agent: int | None,
by_score: bool, gated: bool, judged: str | None) -> tuple[str, list[Any]]:
by_score: bool, gated: bool, judged: str | None,
pay: str | None = None) -> tuple[str, list[Any]]:
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
its parameters in order. The agent's votes are always joined as `v`
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
@@ -569,6 +589,10 @@ class Store:
if judged not in JUDGED:
raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}")
where.append(JUDGED[judged])
if pay:
if pay not in PAY:
raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}")
where.append(PAY[pay])
if open_only:
where.append("closed_at IS NULL")
if hide_muted and agent is not None: