diff --git a/app/jobs/score.py b/app/jobs/score.py
index f0a7773..3b13093 100644
--- a/app/jobs/score.py
+++ b/app/jobs/score.py
@@ -17,7 +17,7 @@ roles out of extract's reach for good.
fit cos(role vector, intent vector), min-max scaled across
the roles this agent scores in this run: the closest is 1,
the farthest 0, a lone role 1
- recency 0.5 ** (days since first_seen / 14)
+ recency 0.5 ** (days since posted_at / 14)
affinity (the agent's affinity for the company + 1) / 3
taste cos(role vector, liked direction), min-max scaled like fit,
where the direction is mean(liked vectors) - mean(passed vectors)
@@ -66,10 +66,13 @@ SEEDED = "agents:seeded"
# ------------------------------------------------------------------- terms
-def recency(first_seen: str, now: datetime) -> float:
- """1.0 for a role first seen now, halving every `HALF_LIFE_DAYS`."""
+def recency(posted_at: str, now: datetime) -> float:
+ """1.0 for a role posted now, halving every `HALF_LIFE_DAYS`. `posted_at`
+ is `app.models.posted_at`: the earlier of when the app first saw the role
+ and, for a trusted host, when the board says it was posted — so a
+ months-old backlog a newly added board brings in does not start at 1.0."""
try:
- seen = datetime.fromisoformat(first_seen)
+ seen = datetime.fromisoformat(posted_at)
except ValueError:
return 0.0
days = max((now - seen).total_seconds(), 0.0) / 86400
@@ -165,7 +168,7 @@ def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
if failed := failed_gates(candidate, gates):
return 0.0, ", ".join(failed)
- terms = [(weights.recency, recency(candidate.first_seen, now)),
+ terms = [(weights.recency, recency(candidate.posted_at, now)),
(weights.affinity, affinity_term(candidate.affinity))]
terms += [(g.weight, 1.0 if state == 1 else 0.0)
for g, state in zip(gates, candidate.gates, strict=True) if g.mode == "prefer"]
diff --git a/app/models.py b/app/models.py
index 1c91b85..46606d0 100644
--- a/app/models.py
+++ b/app/models.py
@@ -228,6 +228,79 @@ STATUSES: tuple[str, ...] = get_args(Status)
AFFINITY = range(-1, 3)
+# -------------------------------------------------------------- posting dates
+
+#: Hosts whose date is a creation or first-publication timestamp, safe to
+#: read as when a role was posted, checked against live boards on
+#: 2026-09-29:
+#:
+#: - `greenhouse`: `plugins.ats._parse_greenhouse` already prefers
+#: `first_published` over `updated_at`. A live board (adyen) showed a job
+#: with `first_published` five months before `updated_at`, proving the
+#: first stays fixed while the second moves on an edit — the resettable
+#: field decisions.md used to warn against is `updated_at`, which this
+#: reads only when `first_published` is missing (rare: 0 of 208 on that
+#: board). The large real gaps this migration measured for Greenhouse
+#: roles (average 143 days between this date and `first_seen`) are further
+#: evidence it holds still rather than tracking every edit.
+#: - `lever` (`createdAt`), `ashby` (`publishedAt`), `personio` (`createdAt`):
+#: named and shaped as creation dates, with no "last edited" field in the
+#: feed at all to reset from, and the same large real gaps as Greenhouse's.
+#: - `smartrecruiters` (`releasedDate`), `recruitee` (`published_at`, falling
+#: back to `created_at`): `releasedDate` reads the same way; a live
+#: Recruitee board (bunq) showed `published_at` and `created_at` minutes
+#: apart while `updated_at` had moved days later on an edited posting,
+#: confirming the field this migration reads is not the one that moves.
+#: - `workable` (`published`): named as a publish date, not an edit one, but
+#: only one live board had postings to check (huggingface, 8 roles) and
+#: none showed a second, "last touched" field to compare against, so this
+#: is trusted on the name and shape alone.
+#: - `jobindex`: the RSS `pubDate` a saved search returns. The live
+#: database's 158 Jobindex-found roles have a `published_at`/`first_seen`
+#: gap of 0.4 to 31 days (median 12), which reads as real, spread-out
+#: posting ages rather than a date that jumps to "now" on every re-crawl.
+#: - `adzuna`: `created`, an aggregator's own record-creation stamp — named
+#: as creation, not modification, like the rest. No live check: this
+#: deployment carries no Adzuna key (`plugins.search`'s own docstring:
+#: "not yet run live") and no Adzuna subscription exists to fetch.
+#:
+#: A host not listed here, or a role whose stored `published_at` fails the
+#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the
+#: safe default this migration is replacing everywhere else.
+TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({
+ "greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
+ "personio", "jobindex", "adzuna",
+})
+
+#: A `published_at` before this is a parsing slip or a placeholder, not a
+#: real posting date: no board here existed before 2015.
+_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
+
+
+def posted_host(raw: dict[str, Any]) -> str:
+ """The host a role's dates came from, read the way `ats` and `search`
+ both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
+ `lever`, ...), `provider` for a cross-company search (`jobindex`,
+ `adzuna`). '' when neither plugin wrote one."""
+ return str(raw.get("board") or raw.get("provider") or "")
+
+
+def posted_at(published_at: str, first_seen: str, host: str) -> str:
+ """The date a role counts as posted: the earlier of `first_seen` (when
+ the app itself first stored it) and `published_at` (what the board
+ claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and
+ `published_at` is neither absurdly old nor later than `first_seen` — a
+ role cannot be posted after the app already saw it, trusted host or not,
+ which also catches a `published_at` the board reports in the future
+ (`first_seen` is never later than the moment this runs). Both dates are
+ `Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
+ is exact and needs no parsing. A role with no trusted date, or with one
+ that fails the check, keeps `first_seen`."""
+ if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen:
+ return published_at
+ return first_seen
+
+
def _json(value: Any, empty: str) -> Any:
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
value that is already parsed passes, and one that does not parse is the
@@ -273,6 +346,10 @@ class Role(Model):
feed_text: str = ""
original_url: str = ""
published_at: str
+ #: The date this role counts as posted: see `posted_at`. Used for
+ #: `recency` and the feed's newest order; `first_seen` stays what it
+ #: always was, when the app itself first stored the role.
+ posted_at: str
first_seen: str
last_seen: str
closed_at: str | None = None
@@ -448,12 +525,12 @@ class Agent(Model):
class Candidate(Model):
- """What scoring reads about one open role for one agent: when it was
- first seen, its company's affinity for that agent, and per gate `1`
- match, `0` no match, `-1` the column is empty."""
+ """What scoring reads about one open role for one agent: its posted
+ date (for `recency`), its company's affinity for that agent, and per
+ gate `1` match, `0` no match, `-1` the column is empty."""
uid: str
- first_seen: str
+ posted_at: str
affinity: int = 0
gates: list[int] = []
diff --git a/app/static/roles.js b/app/static/roles.js
index 2a816aa..8180ea9 100644
--- a/app/static/roles.js
+++ b/app/static/roles.js
@@ -509,7 +509,7 @@
${role.company ? `${esc(role.company)}` : ""}
${role.place ? `${esc(role.place)}` : ""}
${badges(role)}
- ${ago(role.first_seen)}
+ ${ago(role.posted_at)}
${role.gated_reason
? `${esc(role.gated_reason)}`
: ""}
diff --git a/app/storage/schema.py b/app/storage/schema.py
index f9dc207..e670eae 100644
--- a/app/storage/schema.py
+++ b/app/storage/schema.py
@@ -12,10 +12,13 @@ shipped.
from __future__ import annotations
+import json
import logging
import sqlite3
from collections.abc import Callable
+from app.models import posted_at, posted_host
+
log = logging.getLogger(__name__)
SCHEMA = """
@@ -56,9 +59,15 @@ CREATE TABLE IF NOT EXISTS roles (
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
original_url TEXT NOT NULL DEFAULT '', -- the posting page
published_at TEXT NOT NULL, -- what the source claims, UTC
- -- What storing postings buys. first_seen is the only honest posted date,
- -- since Greenhouse resets updated_at on every edit. closed_at is set when a
- -- board fetched in full no longer lists the role, and cleared if it returns.
+ -- What storing postings buys. first_seen is when the app itself first
+ -- stored the role; posted_at (app.models.posted_at) is the earlier of
+ -- first_seen and published_at, trusted only for a host known to report a
+ -- creation date rather than one that moves on an edit
+ -- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
+ -- read posted_at, first_seen stays "when the app found it". closed_at is
+ -- set when a board fetched in full no longer lists the role, and cleared
+ -- if it returns.
+ posted_at TEXT NOT NULL DEFAULT '',
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
closed_at TEXT,
@@ -244,6 +253,11 @@ CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
+-- Serves the newest feed (open_only, ordered by posted_at DESC), which is
+-- most of what `Store.roles` runs: partial, since only an open role's
+-- position in that order is ever asked for.
+CREATE INDEX IF NOT EXISTS idx_roles_posted
+ ON roles(posted_at DESC, uid) WHERE closed_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
CREATE INDEX IF NOT EXISTS idx_application_events
@@ -260,6 +274,23 @@ MARKER = "roles"
Step = str | Callable[[sqlite3.Connection], None]
+
+def _migrate_posted_at(conn: sqlite3.Connection) -> None:
+ """v10's step: add `posted_at`, then backfill it for every row already
+ stored, by the same rule `Store.upsert_role` applies to a new one (see
+ `app.models.posted_at`). A callable, not SQL, because the rule reads
+ `raw` for which host a role came from. `'' NOT NULL` is a placeholder
+ only until the UPDATE below runs, in the same transaction as the ALTER
+ (`_apply` wraps every migration in one); every row gets a real value."""
+ conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''")
+ rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall()
+ conn.executemany(
+ "UPDATE roles SET posted_at = ? WHERE uid = ?",
+ [(posted_at(row["published_at"], row["first_seen"],
+ posted_host(json.loads(row["raw"] or "{}"))), row["uid"])
+ for row in rows])
+
+
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
# Never run as a migration: a file without the marker table is created
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
@@ -344,6 +375,8 @@ ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}';
(9, "companies.ats_slug: the slug of a host the app cannot read yet is kept", """
ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
"""),
+ (10, "roles.posted_at: recency and the newest feed count from when a role "
+ "was posted, not just when the app found it", _migrate_posted_at),
)
#: What a file created from `SCHEMA` today is already at.
diff --git a/app/storage/store.py b/app/storage/store.py
index d62f7fd..0f07f31 100644
--- a/app/storage/store.py
+++ b/app/storage/store.py
@@ -23,7 +23,7 @@ from desk.models import Item
from app.models import (
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key,
- country_of, place_key, remote_of, title_key,
+ country_of, place_key, posted_at, posted_host, remote_of, title_key,
)
from app.storage import schema
@@ -263,6 +263,12 @@ class Store:
from `item.raw` under their own names, with `company` falling back to
the subscription's and `country` to what `place` names; an empty value
never overwrites a filled column.
+
+ `posted_at` (see `app.models.posted_at`) is set from this pull's own
+ `published_at` and `stamp`; on an update the `ON CONFLICT` clause below
+ keeps the earlier of that and what is already stored, so an edit on
+ the board — which can only ever move `published_at` later, never
+ earlier — cannot move `posted_at` later either.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
@@ -270,6 +276,7 @@ class Store:
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
stamp = self.now()
+ published = _iso(item.published_at)
row: dict[str, Any] = {
"uid": item.uid,
"source": item.source,
@@ -284,7 +291,8 @@ class Store:
"description": item.body,
"feed_text": item.excerpt,
"original_url": item.location,
- "published_at": _iso(item.published_at),
+ "published_at": published,
+ "posted_at": posted_at(published, stamp, posted_host(raw)),
"first_seen": stamp,
"last_seen": stamp,
"raw": json.dumps(raw, default=str),
@@ -302,6 +310,7 @@ class Store:
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
+ updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
@@ -429,12 +438,12 @@ class Store:
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
- """Newest first by `first_seen`. `company` and `place` are
+ """Newest first by `posted_at`. `company` and `place` are
case-insensitive substrings of the columns a board wrote — not the
key, so a search does not have to know the exact slug a company folds
to. `q` is matched word by word against `title`; a role must match
every word. `function` and `remote` match a gate column exactly.
- `days` keeps only roles first seen within that many days of now.
+ `days` keeps only roles posted within that many days of now.
`agent` says whose votes, hidden companies and scores are meant. Each
row carries that agent's `vote` and `voted_at`; `judged` (a key of
@@ -471,10 +480,20 @@ class Store:
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
select = "SELECT roles.*, v.vote, v.voted_at"
- order = "roles.first_seen DESC, roles.uid"
- if agent is not None and by_score:
+ order = "roles.posted_at DESC, roles.uid"
+ scored = agent is not None and by_score
+ if scored:
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
order = "score DESC, " + order
+ elif open_only:
+ # Newest first, open only: idx_roles_posted covers this exactly
+ # (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
+ # LIMIT rather than sorting every open role first. Left to the
+ # planner, sqlite3's bundled build (no STAT4) reads the near-100%
+ # selectivity of `closed_at IS NULL` from idx_roles_closed's plain
+ # average and picks that instead, sorting the lot — checked with
+ # EXPLAIN QUERY PLAN on a copy of the live database.
+ sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
found = [ScoredRole.from_row(r) for r in self._rows(
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
by_uid = self._tags_by_uid([r.uid for r in found])
@@ -521,7 +540,7 @@ class Store:
where.append("remote = ?")
params.append(remote)
if days is not None:
- where.append("first_seen >= ?")
+ where.append("posted_at >= ?")
params.append(_iso(self.clock() - timedelta(days=days)))
for word in (q or "").split():
where.append("title LIKE ? ESCAPE '\\'")
@@ -641,7 +660,7 @@ class Store:
return (f"""r.uid IN (SELECT uid FROM (
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
- ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
+ ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
@@ -936,7 +955,7 @@ class Store:
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values
- select = ", ".join(["r.uid", "r.first_seen",
+ select = ", ".join(["r.uid", "r.posted_at",
"COALESCE(ac.affinity, 0) AS affinity", *columns])
rows = self._rows(
f"""SELECT {select} FROM roles r
@@ -944,7 +963,7 @@ class Store:
ON ac.company_key = r.company_key AND ac.agent_id = ?
WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""",
[*params, agent_id])
- return [Candidate(uid=r["uid"], first_seen=r["first_seen"],
+ return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
affinity=int(r["affinity"]),
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
for r in rows]
diff --git a/docs/decisions.md b/docs/decisions.md
index 9547425..5f8dde2 100644
--- a/docs/decisions.md
+++ b/docs/decisions.md
@@ -30,7 +30,9 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**`permit` and `languages` are columns, and so is `extract_tries`.** `permit` answers the plan's open item 1 (visa sponsorship as a gate field) while nothing depends on it yet. newsdesk keeps give-up counts in `raw`; here `upsert_role` rewrites `raw` on every pull, which would reset the count, so the counter is its own column.
-**Postings are persisted.** Storing every posting is what gives `first_seen`, which is the only honest posted date since Greenhouse resets `updated_at` on every edit, and `closed_at`, set by `mark_closed` when a complete fetch of a board no longer lists a role. A role listed again is open again.
+**Postings are persisted.** Storing every posting is what gives `first_seen`, when the app itself first stored the role, and `closed_at`, set by `mark_closed` when a complete fetch of a board no longer lists a role. A role listed again is open again.
+
+**`posted_at` (migration 10) is the earlier of `first_seen` and a trusted host's `published_at`; `first_seen` is left alone, as when the app found the role.** `first_seen` being the only honest posted date was true of Greenhouse's `updated_at`, which resets on every edit; `plugins.ats` already reads `first_published` instead, and that plus the other hosts' dates — Lever's `createdAt`, Ashby's `publishedAt`, SmartRecruiters' `releasedDate`, Recruitee's `published_at`/`created_at`, Personio's `createdAt`, Workable's `published`, and the two `search` providers' (Jobindex's RSS `pubDate`, Adzuna's `created`) — are creation dates, checked against live boards and the live database (`app.models.TRUSTED_POSTED_HOSTS`, with the evidence in its comment). A date in the future or before 2015 is not trusted either, host or no host. Without this, a newly added board's whole backlog scored as freshly posted for two weeks: on the live database, 10,223 of 15,415 board roles were posted more than 30 days before they were first seen (average gap in days: Ashby 78, Greenhouse 143, Lever 226, Personio 333), and open roles with `recency` (`app.jobs.score.recency`) above 0.5 fell from all 15,558 of them to 3,089 (19.9%) once it read `posted_at` instead. `upsert_role` keeps the earliest `posted_at` a role has ever had: an edit on the board can only move its own date later, never earlier, so it can only move `posted_at` earlier too, when the board finally reports the true one.
**The tracker schema is taken from jobhunt-hq (MIT).** `applications` with status in `saved, applied, screen, technical, onsite, offer, rejected, withdrawn`, `next_action`, `next_date`, and an append-only `application_events` written by the store on every status entered. The events table is what "days to first screen" and conversion by source are computed from.
diff --git a/tests/test_score.py b/tests/test_score.py
index 1048e50..05534c8 100644
--- a/tests/test_score.py
+++ b/tests/test_score.py
@@ -80,9 +80,10 @@ def config(tmp_path: Path) -> Config:
def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
- company: str = "Acme", body: str = "", **raw: Any) -> str:
+ company: str = "Acme", body: str = "",
+ published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: Any) -> str:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
- title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
+ title=title, body=body, published_at=published_at,
raw={"company": company, **raw})
item.source = "ats"
store.upsert_role(item)
@@ -274,7 +275,7 @@ def test_recency_halves_every_fourteen_days() -> None:
def test_a_role_without_a_vector_is_scored_without_fit_unless_fit_is_all() -> None:
- candidate = Candidate(uid="u", first_seen=NOW.isoformat(), affinity=0)
+ candidate = Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0)
value, reason = score.score_one(candidate, [], Weights(), None, NOW)
assert reason == "" and value == pytest.approx((0.2 * 1.0 + 0.2 / 3) / 0.4)
@@ -334,7 +335,7 @@ def test_failed_gates_reads_evidence_only() -> None:
Gate(field="seniority", values=["senior"], mode="any"),
Gate(field="function", values=["engineering"], mode="prefer")]
def cand(*states: int) -> Candidate:
- return Candidate(uid="u", first_seen=NOW.isoformat(), affinity=0, gates=list(states))
+ return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0, gates=list(states))
assert score.failed_gates(cand(1, 1, 0, 0), gates) == []
assert score.failed_gates(cand(-1, -1, -1, 0), gates) == []
assert score.failed_gates(cand(0, 0, -1, 1), gates) == [
@@ -402,6 +403,22 @@ def test_an_older_role_scores_lower(
assert scores[new][0] > scores[old][0]
+def test_recency_counts_from_posted_at_not_first_seen(
+ store: Store, config: Config, fake: FakeEmbedder) -> None:
+ """A trusted board's backlog, first seen today, must not score as fresh:
+ recency has to read the board's own (trusted) posted date, not the
+ moment the app happened to pull it in."""
+ stale = role(store, "1", board="greenhouse",
+ published_at=NOW - timedelta(days=28))
+ fresh = role(store, "2") # untrusted host: posted_at falls back to first_seen (NOW)
+ agent = store.save_agent("Any")
+
+ run(config, store, score.run_score)
+
+ scores = store.scores(agent.id)
+ assert scores[fresh][0] > scores[stale][0]
+
+
def test_two_agents_rank_the_same_roles_the_other_way_round(
store: Store, config: Config, fake: FakeEmbedder) -> None:
back = role(store, "1", "Backend Engineer", body="Python services")
diff --git a/tests/test_storage.py b/tests/test_storage.py
index b034c63..a753b97 100644
--- a/tests/test_storage.py
+++ b/tests/test_storage.py
@@ -11,7 +11,10 @@ import pytest
from pydantic import ValidationError
from desk.models import Item
-from app.models import Application, Limits, company_key, place_key, title_key
+from app.models import (
+ Application, Limits, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at,
+ posted_host, title_key,
+)
from app.storage import LATEST, Store
@@ -38,10 +41,11 @@ def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
yield s
-def posting(source_id: str = "123", **raw: object) -> Item:
+def posting(source_id: str = "123", *,
+ published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/acme/{source_id}",
title="Senior Backend Engineer (m/w/d)", body="We build things.",
- published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
+ published_at=published_at, raw=dict(raw))
item.source = "ats"
return item
@@ -62,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
- assert LATEST == 9
+ assert LATEST == 10
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -86,8 +90,9 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
Store(fresh)
Store(old)
with sqlite3.connect(old) as conn:
+ conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
for column in ("permit", "languages", "extract_tries", "summary", "summarised_at",
- "article_text", "article_at", "read_at"):
+ "article_text", "article_at", "read_at", "posted_at"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
for column in ("intent_key", "intent_vec", "limits"): # v3, v8
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
@@ -112,11 +117,13 @@ def test_a_v8_file_gains_ats_slug_and_keeps_its_companies(tmp_path: Path) -> Non
Store(path).add_company("Acme", ats="teamtailor", country="DK")
with sqlite3.connect(path) as conn:
conn.execute("ALTER TABLE companies DROP COLUMN ats_slug")
+ conn.execute("DROP INDEX idx_roles_posted") # v10, which follows
+ conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
conn.execute("PRAGMA user_version = 8")
migrated = Store(path)
- assert migrated.migrations_applied == [9]
+ assert migrated.migrations_applied == [9, 10]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.country, acme.ats_slug) == ("teamtailor", "DK", "")
@@ -127,6 +134,8 @@ def _downgrade_to_v4(path: Path) -> None:
the company, no read state, no tags, no per-agent tables."""
with sqlite3.connect(path) as conn:
conn.execute("ALTER TABLE roles DROP COLUMN read_at")
+ conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
+ conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9
conn.execute("DROP TABLE role_tags")
conn.execute("ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1))")
conn.execute("ALTER TABLE roles ADD COLUMN voted_at TEXT")
@@ -220,6 +229,34 @@ def test_a_v4_file_migrates_read_state_and_tags(tmp_path: Path) -> None:
assert "role_tags" in tables
+def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
+ tmp_path: Path, clock: Clock) -> None:
+ path = tmp_path / "v9.db"
+ store = Store(path)
+ store.clock = clock
+ old_date = datetime(2020, 1, 1, tzinfo=UTC)
+ trusted = posting("1", board="greenhouse", published_at=old_date)
+ untrusted = posting("2", published_at=old_date)
+ store.upsert_role(trusted)
+ store.upsert_role(untrusted)
+ before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid)
+ assert before_trusted is not None and before_untrusted is not None
+
+ with sqlite3.connect(path) as conn:
+ conn.execute("DROP INDEX idx_roles_posted")
+ conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
+ conn.execute("PRAGMA user_version = 9")
+
+ migrated = Store(path)
+
+ assert migrated.migrations_applied == [10]
+ after_trusted, after_untrusted = migrated.role(trusted.uid), migrated.role(untrusted.uid)
+ assert after_trusted is not None and after_untrusted is not None
+ # The backfill reproduces exactly what upsert_role would have written.
+ assert after_trusted.posted_at == before_trusted.posted_at == "2020-01-01T00:00:00+00:00"
+ assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen
+
+
def test_state_is_a_mapping(store: Store) -> None:
store.state["ats:last"] = "2026-09-23"
assert store.state["ats:last"] == "2026-09-23"
@@ -240,6 +277,37 @@ def test_keys() -> None:
assert company_key("Celonis SE") == company_key("celonis") == "celonis"
+# -------------------------------------------------------------- posted_at
+
+def test_posted_host_reads_board_then_provider() -> None:
+ assert posted_host({"board": "greenhouse", "provider": "jobindex"}) == "greenhouse"
+ assert posted_host({"provider": "jobindex"}) == "jobindex"
+ assert posted_host({}) == ""
+
+
+def test_posted_at_trusts_only_a_known_host() -> None:
+ assert "greenhouse" in TRUSTED_POSTED_HOSTS and "workday" not in TRUSTED_POSTED_HOSTS
+ early = "2020-01-01T00:00:00+00:00"
+ late = "2026-09-23T09:00:00+00:00"
+ # A trusted host's earlier date wins over first_seen...
+ assert posted_at(early, late, "greenhouse") == early
+ # ...but an untrusted (or unknown) host never moves it off first_seen.
+ assert posted_at(early, late, "workday") == late
+ assert posted_at(early, late, "") == late
+
+
+def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
+ first_seen = "2026-09-23T09:00:00+00:00"
+ future = "2026-09-24T00:00:00+00:00" # after first_seen
+ absurd = "2014-12-31T23:59:59+00:00" # before the 2015 floor
+ assert posted_at(future, first_seen, "greenhouse") == first_seen
+ assert posted_at(absurd, first_seen, "greenhouse") == first_seen
+ # The floor itself, and the instant first_seen, both still count.
+ assert posted_at("2015-01-01T00:00:00+00:00", first_seen, "greenhouse") == \
+ "2015-01-01T00:00:00+00:00"
+ assert posted_at(first_seen, first_seen, "greenhouse") == first_seen
+
+
# ------------------------------------------------------------------- roles
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
@@ -274,6 +342,39 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
+def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest(
+ store: Store, clock: Clock) -> None:
+ """A board's first pull can carry a stale, too-recent published_at (a
+ board that has not backfilled first_published yet, say); a later pull
+ that finally reports the true, earlier date must still win — an edit on
+ the board only ever moves published_at later, never earlier, so the
+ earliest posted_at ever computed is the one to keep."""
+ store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20,
+ tzinfo=UTC)))
+ first = store.role(posting().uid)
+ assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00"
+
+ clock.tick()
+ store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1,
+ tzinfo=UTC)))
+ earlier = store.role(posting().uid)
+ assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00"
+
+ clock.tick()
+ # A later, LATER published_at (an edit) must not push posted_at forward.
+ store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25,
+ tzinfo=UTC)))
+ still_earlier = store.role(posting().uid)
+ assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00"
+
+
+def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None:
+ store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
+ role = store.role(posting().uid)
+ assert role is not None
+ assert role.posted_at == role.first_seen
+
+
def test_company_falls_back_to_the_subscription(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
store.upsert_role(posting(), sub)
@@ -294,6 +395,58 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
assert [r.source_id for r in store.roles(limit=1)] == ["3"]
+def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None:
+ """A role from a trusted board found today but posted months ago must not
+ outrank one the app has genuinely known about for longer."""
+ store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen
+ clock.tick()
+ store.upsert_role(posting("old", board="greenhouse",
+ published_at=datetime(2020, 1, 1, tzinfo=UTC)))
+ assert [r.source_id for r in store.roles()] == ["recent", "old"]
+ assert [r.source_id for r in store.roles(days=30)] == ["recent"]
+
+
+def _spy_sql(monkeypatch: pytest.MonkeyPatch) -> list[str]:
+ """Every SQL statement a `Store` runs through `_rows`, in order."""
+ captured: list[str] = []
+ original = Store._rows
+
+ def spy(self: Store, sql: str, params: object = ()) -> list[sqlite3.Row]:
+ captured.append(sql)
+ return original(self, sql, params)
+
+ monkeypatch.setattr(Store, "_rows", spy)
+ return captured
+
+
+def test_the_newest_feed_query_uses_the_partial_posted_at_index(
+ store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ """`Store.roles`' default order (newest, open only) is forced onto
+ `idx_roles_posted`: left to the planner's plain, non-STAT4 statistics, it
+ can instead pick `idx_roles_closed` and sort every open role, which on
+ the live database (15,558 of 15,815 roles open) measured over 30x slower
+ than the plan this asserts (see docs/decisions.md)."""
+ store.upsert_role(posting("1"))
+ sql, _ = store._roles_from(
+ company=None, place=None, open_only=True, q=None, function=None, remote=None,
+ days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any",
+ agent=None, by_score=True, gated=False, judged=None)
+ assert "INDEXED BY" not in sql # the hint belongs to roles(), not the shared builder
+
+ captured = _spy_sql(monkeypatch)
+ store.roles(limit=10)
+ assert "INDEXED BY idx_roles_posted" in captured[0]
+ with sqlite3.connect(store.path) as conn:
+ plan = " ".join(row[3] for row in
+ conn.execute("EXPLAIN QUERY PLAN " + captured[0], [None, 10, 0]))
+ assert "USING INDEX idx_roles_posted" in plan and "TEMP B-TREE" not in plan
+
+ # open_only=False cannot use it: the index is partial on closed_at IS NULL.
+ captured.clear()
+ store.roles(limit=10, open_only=False)
+ assert "INDEXED BY" not in captured[0]
+
+
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin",
function="engineering", remote=True))