Date a role by posted_at, not just first_seen

recency and the newest feed both counted from first_seen, when the app
itself first stored a role, so a newly added board's whole backlog scored
as freshly posted for two weeks: 10,223 of 15,415 board roles were posted
more than 30 days before they were first seen (average gap in days: Ashby
78, Greenhouse 143, Lever 226, Personio 333).

Add roles.posted_at (migration 9): the earlier of first_seen and a trusted
host's published_at, where TRUSTED_POSTED_HOSTS (app/models.py) records
which of the ats and search hosts report a creation date rather than one
that moves on an edit, checked against live boards and the live database.
A future or pre-2015 date is not trusted either. upsert_role sets it on
insert and keeps the earliest on update, so an edit on the board can only
move it earlier, never later. recency, the feed's newest order, the "new
since" days filter and _top_scope's tie-break now read posted_at;
first_seen is unchanged and still what the UI shows as when the app found
a role. The row's age in roles.js now shows the posted date instead.

A partial index (posted_at DESC, uid) WHERE closed_at IS NULL serves the
newest feed; sqlite3's bundled build has no STAT4, so its planner
otherwise sorts every open role instead of using it, measured 30x slower
on the live database, so Store.roles() forces it for that query shape.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet committed 2026-09-29 14:13:06 +02:00
1 parent 0558ef19ed
commit 75eca1c914
8 files changed
+335 -33

No files matched your search

+36 -3
View File
@@ -12,10 +12,13 @@ shipped.
from __future__ import annotations
import json
import logging
import sqlite3
from collections.abc import Callable
from app.models import posted_at, posted_host
log = logging.getLogger(__name__)
SCHEMA = """
@@ -56,9 +59,15 @@ CREATE TABLE IF NOT EXISTS roles (
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
original_url TEXT NOT NULL DEFAULT '', -- the posting page
published_at TEXT NOT NULL, -- what the source claims, UTC
-- What storing postings buys. first_seen is the only honest posted date,
-- since Greenhouse resets updated_at on every edit. closed_at is set when a
-- board fetched in full no longer lists the role, and cleared if it returns.
-- What storing postings buys. first_seen is when the app itself first
-- stored the role; posted_at (app.models.posted_at) is the earlier of
-- first_seen and published_at, trusted only for a host known to report a
-- creation date rather than one that moves on an edit
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
-- read posted_at, first_seen stays "when the app found it". closed_at is
-- set when a board fetched in full no longer lists the role, and cleared
-- if it returns.
posted_at TEXT NOT NULL DEFAULT '',
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
closed_at TEXT,
@@ -241,6 +250,11 @@ CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
-- Serves the newest feed (open_only, ordered by posted_at DESC), which is
-- most of what `Store.roles` runs: partial, since only an open role's
-- position in that order is ever asked for.
CREATE INDEX IF NOT EXISTS idx_roles_posted
ON roles(posted_at DESC, uid) WHERE closed_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
CREATE INDEX IF NOT EXISTS idx_application_events
@@ -257,6 +271,23 @@ MARKER = "roles"
Step = str | Callable[[sqlite3.Connection], None]
def _migrate_posted_at(conn: sqlite3.Connection) -> None:
"""v9's step: add `posted_at`, then backfill it for every row already
stored, by the same rule `Store.upsert_role` applies to a new one (see
`app.models.posted_at`). A callable, not SQL, because the rule reads
`raw` for which host a role came from. `'' NOT NULL` is a placeholder
only until the UPDATE below runs, in the same transaction as the ALTER
(`_apply` wraps every migration in one); every row gets a real value."""
conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''")
rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall()
conn.executemany(
"UPDATE roles SET posted_at = ? WHERE uid = ?",
[(posted_at(row["published_at"], row["first_seen"],
posted_host(json.loads(row["raw"] or "{}"))), row["uid"])
for row in rows])
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
# Never run as a migration: a file without the marker table is created
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
@@ -338,6 +369,8 @@ ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT '';
(8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """
ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}';
"""),
(9, "roles.posted_at: recency and the newest feed count from when a role "
"was posted, not just when the app found it", _migrate_posted_at),
)
#: What a file created from `SCHEMA` today is already at.
+29 -10
View File
@@ -23,7 +23,7 @@ from desk.models import Item
from app.models import (
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key,
country_of, place_key, remote_of, title_key,
country_of, place_key, posted_at, posted_host, remote_of, title_key,
)
from app.storage import schema
@@ -262,6 +262,12 @@ class Store:
from `item.raw` under their own names, with `company` falling back to
the subscription's and `country` to what `place` names; an empty value
never overwrites a filled column.
`posted_at` (see `app.models.posted_at`) is set from this pull's own
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
keeps the earlier of that and what is already stored, so an edit on
the board — which can only ever move `published_at` later, never
earlier — cannot move `posted_at` later either.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
@@ -269,6 +275,7 @@ class Store:
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
stamp = self.now()
published = _iso(item.published_at)
row: dict[str, Any] = {
"uid": item.uid,
"source": item.source,
@@ -283,7 +290,8 @@ class Store:
"description": item.body,
"feed_text": item.excerpt,
"original_url": item.location,
"published_at": _iso(item.published_at),
"published_at": published,
"posted_at": posted_at(published, stamp, posted_host(raw)),
"first_seen": stamp,
"last_seen": stamp,
"raw": json.dumps(raw, default=str),
@@ -301,6 +309,7 @@ class Store:
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
@@ -428,12 +437,12 @@ class Store:
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
"""Newest first by `first_seen`. `company` and `place` are
"""Newest first by `posted_at`. `company` and `place` are
case-insensitive substrings of the columns a board wrote — not the
key, so a search does not have to know the exact slug a company folds
to. `q` is matched word by word against `title`; a role must match
every word. `function` and `remote` match a gate column exactly.
`days` keeps only roles first seen within that many days of now.
`days` keeps only roles posted within that many days of now.
`agent` says whose votes, hidden companies and scores are meant. Each
row carries that agent's `vote` and `voted_at`; `judged` (a key of
@@ -470,10 +479,20 @@ class Store:
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
select = "SELECT roles.*, v.vote, v.voted_at"
order = "roles.first_seen DESC, roles.uid"
if agent is not None and by_score:
order = "roles.posted_at DESC, roles.uid"
scored = agent is not None and by_score
if scored:
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
order = "score DESC, " + order
elif open_only:
# Newest first, open only: idx_roles_posted covers this exactly
# (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
# LIMIT rather than sorting every open role first. Left to the
# planner, sqlite3's bundled build (no STAT4) reads the near-100%
# selectivity of `closed_at IS NULL` from idx_roles_closed's plain
# average and picks that instead, sorting the lot — checked with
# EXPLAIN QUERY PLAN on a copy of the live database.
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
found = [ScoredRole.from_row(r) for r in self._rows(
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
by_uid = self._tags_by_uid([r.uid for r in found])
@@ -520,7 +539,7 @@ class Store:
where.append("remote = ?")
params.append(remote)
if days is not None:
where.append("first_seen >= ?")
where.append("posted_at >= ?")
params.append(_iso(self.clock() - timedelta(days=days)))
for word in (q or "").split():
where.append("title LIKE ? ESCAPE '\\'")
@@ -640,7 +659,7 @@ class Store:
return (f"""r.uid IN (SELECT uid FROM (
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
@@ -935,7 +954,7 @@ class Store:
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values
select = ", ".join(["r.uid", "r.first_seen",
select = ", ".join(["r.uid", "r.posted_at",
"COALESCE(ac.affinity, 0) AS affinity", *columns])
rows = self._rows(
f"""SELECT {select} FROM roles r
@@ -943,7 +962,7 @@ class Store:
ON ac.company_key = r.company_key AND ac.agent_id = ?
WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""",
[*params, agent_id])
return [Candidate(uid=r["uid"], first_seen=r["first_seen"],
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
affinity=int(r["affinity"]),
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
for r in rows]