Date a role by posted_at, not just first_seen

recency and the newest feed both counted from first_seen, when the app
itself first stored a role, so a newly added board's whole backlog scored
as freshly posted for two weeks: 10,223 of 15,415 board roles were posted
more than 30 days before they were first seen (average gap in days: Ashby
78, Greenhouse 143, Lever 226, Personio 333).

Add roles.posted_at (migration 9): the earlier of first_seen and a trusted
host's published_at, where TRUSTED_POSTED_HOSTS (app/models.py) records
which of the ats and search hosts report a creation date rather than one
that moves on an edit, checked against live boards and the live database.
A future or pre-2015 date is not trusted either. upsert_role sets it on
insert and keeps the earliest on update, so an edit on the board can only
move it earlier, never later. recency, the feed's newest order, the "new
since" days filter and _top_scope's tie-break now read posted_at;
first_seen is unchanged and still what the UI shows as when the app found
a role. The row's age in roles.js now shows the posted date instead.

A partial index (posted_at DESC, uid) WHERE closed_at IS NULL serves the
newest feed; sqlite3's bundled build has no STAT4, so its planner
otherwise sorts every open role instead of using it, measured 30x slower
on the live database, so Store.roles() forces it for that query shape.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet committed 2026-09-29 14:13:06 +02:00
1 parent 0558ef19ed
commit 75eca1c914
8 files changed
+335 -33

No files matched your search

+156 -5
View File
@@ -11,7 +11,10 @@ import pytest
from pydantic import ValidationError
from desk.models import Item
from app.models import Application, Limits, company_key, place_key, title_key
from app.models import (
Application, Limits, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at,
posted_host, title_key,
)
from app.storage import LATEST, Store
@@ -38,10 +41,11 @@ def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
yield s
def posting(source_id: str = "123", **raw: object) -> Item:
def posting(source_id: str = "123", *,
published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/acme/{source_id}",
title="Senior Backend Engineer (m/w/d)", body="We build things.",
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
published_at=published_at, raw=dict(raw))
item.source = "ats"
return item
@@ -62,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 8
assert LATEST == 9
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -86,8 +90,9 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
Store(fresh)
Store(old)
with sqlite3.connect(old) as conn:
conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
for column in ("permit", "languages", "extract_tries", "summary", "summarised_at",
"article_text", "article_at", "read_at"):
"article_text", "article_at", "read_at", "posted_at"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
for column in ("intent_key", "intent_vec", "limits"): # v3, v8
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
@@ -112,6 +117,8 @@ def _downgrade_to_v4(path: Path) -> None:
the company, no read state, no tags, no per-agent tables."""
with sqlite3.connect(path) as conn:
conn.execute("ALTER TABLE roles DROP COLUMN read_at")
conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9
conn.execute("DROP TABLE role_tags")
conn.execute("ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1))")
conn.execute("ALTER TABLE roles ADD COLUMN voted_at TEXT")
@@ -205,6 +212,34 @@ def test_a_v4_file_migrates_read_state_and_tags(tmp_path: Path) -> None:
assert "role_tags" in tables
def test_v9_backfills_posted_at_by_the_same_rule_upsert_uses(
tmp_path: Path, clock: Clock) -> None:
path = tmp_path / "v8.db"
store = Store(path)
store.clock = clock
old_date = datetime(2020, 1, 1, tzinfo=UTC)
trusted = posting("1", board="greenhouse", published_at=old_date)
untrusted = posting("2", published_at=old_date)
store.upsert_role(trusted)
store.upsert_role(untrusted)
before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid)
assert before_trusted is not None and before_untrusted is not None
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_posted")
conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
conn.execute("PRAGMA user_version = 8")
migrated = Store(path)
assert migrated.migrations_applied == [9]
after_trusted, after_untrusted = migrated.role(trusted.uid), migrated.role(untrusted.uid)
assert after_trusted is not None and after_untrusted is not None
# The backfill reproduces exactly what upsert_role would have written.
assert after_trusted.posted_at == before_trusted.posted_at == "2020-01-01T00:00:00+00:00"
assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen
def test_state_is_a_mapping(store: Store) -> None:
store.state["ats:last"] = "2026-09-23"
assert store.state["ats:last"] == "2026-09-23"
@@ -225,6 +260,37 @@ def test_keys() -> None:
assert company_key("Celonis SE") == company_key("celonis") == "celonis"
# -------------------------------------------------------------- posted_at
def test_posted_host_reads_board_then_provider() -> None:
assert posted_host({"board": "greenhouse", "provider": "jobindex"}) == "greenhouse"
assert posted_host({"provider": "jobindex"}) == "jobindex"
assert posted_host({}) == ""
def test_posted_at_trusts_only_a_known_host() -> None:
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "workday" not in TRUSTED_POSTED_HOSTS
early = "2020-01-01T00:00:00+00:00"
late = "2026-09-23T09:00:00+00:00"
# A trusted host's earlier date wins over first_seen...
assert posted_at(early, late, "greenhouse") == early
# ...but an untrusted (or unknown) host never moves it off first_seen.
assert posted_at(early, late, "workday") == late
assert posted_at(early, late, "") == late
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
first_seen = "2026-09-23T09:00:00+00:00"
future = "2026-09-24T00:00:00+00:00" # after first_seen
absurd = "2014-12-31T23:59:59+00:00" # before the 2015 floor
assert posted_at(future, first_seen, "greenhouse") == first_seen
assert posted_at(absurd, first_seen, "greenhouse") == first_seen
# The floor itself, and the instant first_seen, both still count.
assert posted_at("2015-01-01T00:00:00+00:00", first_seen, "greenhouse") == \
"2015-01-01T00:00:00+00:00"
assert posted_at(first_seen, first_seen, "greenhouse") == first_seen
# ------------------------------------------------------------------- roles
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
@@ -259,6 +325,39 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest(
store: Store, clock: Clock) -> None:
"""A board's first pull can carry a stale, too-recent published_at (a
board that has not backfilled first_published yet, say); a later pull
that finally reports the true, earlier date must still win — an edit on
the board only ever moves published_at later, never earlier, so the
earliest posted_at ever computed is the one to keep."""
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20,
tzinfo=UTC)))
first = store.role(posting().uid)
assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00"
clock.tick()
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1,
tzinfo=UTC)))
earlier = store.role(posting().uid)
assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00"
clock.tick()
# A later, LATER published_at (an edit) must not push posted_at forward.
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25,
tzinfo=UTC)))
still_earlier = store.role(posting().uid)
assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00"
def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None:
store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
role = store.role(posting().uid)
assert role is not None
assert role.posted_at == role.first_seen
def test_company_falls_back_to_the_subscription(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
store.upsert_role(posting(), sub)
@@ -279,6 +378,58 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
assert [r.source_id for r in store.roles(limit=1)] == ["3"]
def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None:
"""A role from a trusted board found today but posted months ago must not
outrank one the app has genuinely known about for longer."""
store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen
clock.tick()
store.upsert_role(posting("old", board="greenhouse",
published_at=datetime(2020, 1, 1, tzinfo=UTC)))
assert [r.source_id for r in store.roles()] == ["recent", "old"]
assert [r.source_id for r in store.roles(days=30)] == ["recent"]
def _spy_sql(monkeypatch: pytest.MonkeyPatch) -> list[str]:
"""Every SQL statement a `Store` runs through `_rows`, in order."""
captured: list[str] = []
original = Store._rows
def spy(self: Store, sql: str, params: object = ()) -> list[sqlite3.Row]:
captured.append(sql)
return original(self, sql, params)
monkeypatch.setattr(Store, "_rows", spy)
return captured
def test_the_newest_feed_query_uses_the_partial_posted_at_index(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""`Store.roles`' default order (newest, open only) is forced onto
`idx_roles_posted`: left to the planner's plain, non-STAT4 statistics, it
can instead pick `idx_roles_closed` and sort every open role, which on
the live database (15,558 of 15,815 roles open) measured over 30x slower
than the plan this asserts (see docs/decisions.md)."""
store.upsert_role(posting("1"))
sql, _ = store._roles_from(
company=None, place=None, open_only=True, q=None, function=None, remote=None,
days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any",
agent=None, by_score=True, gated=False, judged=None)
assert "INDEXED BY" not in sql # the hint belongs to roles(), not the shared builder
captured = _spy_sql(monkeypatch)
store.roles(limit=10)
assert "INDEXED BY idx_roles_posted" in captured[0]
with sqlite3.connect(store.path) as conn:
plan = " ".join(row[3] for row in
conn.execute("EXPLAIN QUERY PLAN " + captured[0], [None, 10, 0]))
assert "USING INDEX idx_roles_posted" in plan and "TEMP B-TREE" not in plan
# open_only=False cannot use it: the index is partial on closed_at IS NULL.
captured.clear()
store.roles(limit=10, open_only=False)
assert "INDEXED BY" not in captured[0]
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin",
function="engineering", remote=True))