Keep a board's stated pay as salary_text, even when it does not normalise
Every ats parser now writes one human-readable pay line whether or not it
converts to yearly salary_min/salary_max/currency: hourly and daily rates,
Greenhouse's several pay zones, and a range with no currency all survive as
text instead of being dropped outright. Stored in roles.salary_text
(migration 12), replaced whole on every pull like title, so a board that
stops stating pay clears it rather than leaving a stale line. salary_state
is derived, not stored ('normalised', 'stated', or ''), and is what the
feed's new Pay filter and badge read; gates and scoring are unchanged, still
reading only the normalised columns.
Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
fa4e1a657f
18 files changed
+528
-57
No files matched your search
@@ -86,7 +86,8 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- whose description is only a teaser.
|
||||
article_text TEXT NOT NULL DEFAULT '',
|
||||
article_at TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
created_at TEXT NOT NULL,
|
||||
salary_text TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- The open vocabulary summarise reads off a posting: the role family, the
|
||||
@@ -381,6 +382,10 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
||||
# its current rules is the whole point (see _recompute_function below).
|
||||
(11, "roles.function recomputed for every not-yet-extracted role, word-boundary "
|
||||
"classifier fix", lambda conn: _recompute_function(conn)),
|
||||
(12, "roles.salary_text: the pay a board stated as one line, kept even when it "
|
||||
"could not be normalised into salary_min/salary_max/currency", """
|
||||
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
|
||||
+31
-7
@@ -60,6 +60,17 @@ JUDGED: dict[str, str] = {
|
||||
"passed": "v.vote = -1",
|
||||
}
|
||||
|
||||
#: The feed's Pay filter, `_roles_from`'s `pay`: the SQL that is true for
|
||||
#: each choice, over `roles`. `salary_state` (`app.models.Role`) is derived,
|
||||
#: not stored, so the filter spells the same rule out in SQL rather than
|
||||
#: reading a column: 'normalised' is `salary_min` or `salary_max` set,
|
||||
#: 'stated' is `salary_text` set and neither of those is.
|
||||
PAY: dict[str, str] = {
|
||||
"stated": "(salary_min IS NOT NULL OR salary_max IS NOT NULL OR salary_text != '')",
|
||||
"normalised": "(salary_min IS NOT NULL OR salary_max IS NOT NULL)",
|
||||
"stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)",
|
||||
}
|
||||
|
||||
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
|
||||
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
|
||||
#: moves the role at once and the stored score stays what `score` computed.
|
||||
@@ -313,6 +324,10 @@ class Store:
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
# A board's own pay line, one per pull like title or published_at —
|
||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||
# must be able to clear it, not leave a stale line behind.
|
||||
row["salary_text"] = str(raw.get("salary_text") or "")
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", *_RAW_TEXT)
|
||||
@@ -320,7 +335,8 @@ class Store:
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
||||
"salary_text")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
@@ -457,13 +473,15 @@ class Store:
|
||||
hide_muted: bool = False, include_read: bool = True,
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
|
||||
judged: str | None = None, pay: str | None = None,
|
||||
offset: int = 0) -> list[ScoredRole]:
|
||||
"""Newest first by `posted_at`. `company` and `place` are
|
||||
case-insensitive substrings of the columns a board wrote — not the
|
||||
key, so a search does not have to know the exact slug a company folds
|
||||
to. `q` is matched word by word against `title`; a role must match
|
||||
every word. `function` and `remote` match a gate column exactly.
|
||||
`days` keeps only roles posted within that many days of now.
|
||||
`days` keeps only roles posted within that many days of now. `pay`
|
||||
(a key of `PAY`) filters on `salary_state`.
|
||||
|
||||
`agent` says whose votes, hidden companies and scores are meant. Each
|
||||
row carries that agent's `vote` and `voted_at`; `judged` (a key of
|
||||
@@ -498,7 +516,7 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged)
|
||||
judged=judged, pay=pay)
|
||||
select = "SELECT roles.*, v.vote, v.voted_at"
|
||||
order = "roles.posted_at DESC, roles.uid"
|
||||
scored = agent is not None and by_score
|
||||
@@ -527,13 +545,14 @@ class Store:
|
||||
days: int | None = None, hide_muted: bool = False,
|
||||
include_read: bool = True, tags: Sequence[str] = (),
|
||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||
gated: bool = False, judged: str | None = None) -> int:
|
||||
gated: bool = False, judged: str | None = None,
|
||||
pay: str | None = None) -> int:
|
||||
"""How many roles `roles` would list with the same filters and no limit."""
|
||||
sql, params = self._roles_from(
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged)
|
||||
judged=judged, pay=pay)
|
||||
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
@@ -541,7 +560,8 @@ class Store:
|
||||
q: str | None, function: str | None, remote: str | None,
|
||||
days: int | None, hide_muted: bool, include_read: bool,
|
||||
tags: Sequence[str], tag_mode: str, agent: int | None,
|
||||
by_score: bool, gated: bool, judged: str | None) -> tuple[str, list[Any]]:
|
||||
by_score: bool, gated: bool, judged: str | None,
|
||||
pay: str | None = None) -> tuple[str, list[Any]]:
|
||||
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
||||
its parameters in order. The agent's votes are always joined as `v`
|
||||
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
|
||||
@@ -569,6 +589,10 @@ class Store:
|
||||
if judged not in JUDGED:
|
||||
raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}")
|
||||
where.append(JUDGED[judged])
|
||||
if pay:
|
||||
if pay not in PAY:
|
||||
raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}")
|
||||
where.append(PAY[pay])
|
||||
if open_only:
|
||||
where.append("closed_at IS NULL")
|
||||
if hide_muted and agent is not None:
|
||||
|
||||
Reference in new issue
Block a user