From 75eca1c91409edda3c62e11d1dfcc5531006c29b Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 14:13:06 +0200 Subject: [PATCH] Date a role by posted_at, not just first_seen recency and the newest feed both counted from first_seen, when the app itself first stored a role, so a newly added board's whole backlog scored as freshly posted for two weeks: 10,223 of 15,415 board roles were posted more than 30 days before they were first seen (average gap in days: Ashby 78, Greenhouse 143, Lever 226, Personio 333). Add roles.posted_at (migration 9): the earlier of first_seen and a trusted host's published_at, where TRUSTED_POSTED_HOSTS (app/models.py) records which of the ats and search hosts report a creation date rather than one that moves on an edit, checked against live boards and the live database. A future or pre-2015 date is not trusted either. upsert_role sets it on insert and keeps the earliest on update, so an edit on the board can only move it earlier, never later. recency, the feed's newest order, the "new since" days filter and _top_scope's tie-break now read posted_at; first_seen is unchanged and still what the UI shows as when the app found a role. The row's age in roles.js now shows the posted date instead. A partial index (posted_at DESC, uid) WHERE closed_at IS NULL serves the newest feed; sqlite3's bundled build has no STAT4, so its planner otherwise sorts every open role instead of using it, measured 30x slower on the live database, so Store.roles() forces it for that query shape. Co-Authored-By: Claude Sonnet --- app/jobs/score.py | 13 ++-- app/models.py | 85 ++++++++++++++++++++-- app/static/roles.js | 2 +- app/storage/schema.py | 39 +++++++++- app/storage/store.py | 39 +++++++--- docs/decisions.md | 4 +- tests/test_score.py | 25 +++++-- tests/test_storage.py | 161 ++++++++++++++++++++++++++++++++++++++++-- 8 files changed, 335 insertions(+), 33 deletions(-) diff --git a/app/jobs/score.py b/app/jobs/score.py index cf02428..cf11fc0 100644 --- a/app/jobs/score.py +++ b/app/jobs/score.py @@ -17,7 +17,7 @@ roles out of extract's reach for good. fit cos(role vector, intent vector), min-max scaled across the roles this agent scores in this run: the closest is 1, the farthest 0, a lone role 1 - recency 0.5 ** (days since first_seen / 14) + recency 0.5 ** (days since posted_at / 14) affinity (the agent's affinity for the company + 1) / 3 taste cos(role vector, liked direction), min-max scaled like fit, where the direction is mean(liked vectors) - mean(passed vectors) @@ -66,10 +66,13 @@ SEEDED = "agents:seeded" # ------------------------------------------------------------------- terms -def recency(first_seen: str, now: datetime) -> float: - """1.0 for a role first seen now, halving every `HALF_LIFE_DAYS`.""" +def recency(posted_at: str, now: datetime) -> float: + """1.0 for a role posted now, halving every `HALF_LIFE_DAYS`. `posted_at` + is `app.models.posted_at`: the earlier of when the app first saw the role + and, for a trusted host, when the board says it was posted — so a + months-old backlog a newly added board brings in does not start at 1.0.""" try: - seen = datetime.fromisoformat(first_seen) + seen = datetime.fromisoformat(posted_at) except ValueError: return 0.0 days = max((now - seen).total_seconds(), 0.0) / 86400 @@ -164,7 +167,7 @@ def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights, if failed := failed_gates(candidate, gates): return 0.0, ", ".join(failed) - terms = [(weights.recency, recency(candidate.first_seen, now)), + terms = [(weights.recency, recency(candidate.posted_at, now)), (weights.affinity, affinity_term(candidate.affinity))] terms += [(g.weight, 1.0 if state == 1 else 0.0) for g, state in zip(gates, candidate.gates, strict=True) if g.mode == "prefer"] diff --git a/app/models.py b/app/models.py index 70d44e6..bb3d22e 100644 --- a/app/models.py +++ b/app/models.py @@ -228,6 +228,79 @@ STATUSES: tuple[str, ...] = get_args(Status) AFFINITY = range(-1, 3) +# -------------------------------------------------------------- posting dates + +#: Hosts whose date is a creation or first-publication timestamp, safe to +#: read as when a role was posted, checked against live boards on +#: 2026-09-29: +#: +#: - `greenhouse`: `plugins.ats._parse_greenhouse` already prefers +#: `first_published` over `updated_at`. A live board (adyen) showed a job +#: with `first_published` five months before `updated_at`, proving the +#: first stays fixed while the second moves on an edit — the resettable +#: field decisions.md used to warn against is `updated_at`, which this +#: reads only when `first_published` is missing (rare: 0 of 208 on that +#: board). The large real gaps this migration measured for Greenhouse +#: roles (average 143 days between this date and `first_seen`) are further +#: evidence it holds still rather than tracking every edit. +#: - `lever` (`createdAt`), `ashby` (`publishedAt`), `personio` (`createdAt`): +#: named and shaped as creation dates, with no "last edited" field in the +#: feed at all to reset from, and the same large real gaps as Greenhouse's. +#: - `smartrecruiters` (`releasedDate`), `recruitee` (`published_at`, falling +#: back to `created_at`): `releasedDate` reads the same way; a live +#: Recruitee board (bunq) showed `published_at` and `created_at` minutes +#: apart while `updated_at` had moved days later on an edited posting, +#: confirming the field this migration reads is not the one that moves. +#: - `workable` (`published`): named as a publish date, not an edit one, but +#: only one live board had postings to check (huggingface, 8 roles) and +#: none showed a second, "last touched" field to compare against, so this +#: is trusted on the name and shape alone. +#: - `jobindex`: the RSS `pubDate` a saved search returns. The live +#: database's 158 Jobindex-found roles have a `published_at`/`first_seen` +#: gap of 0.4 to 31 days (median 12), which reads as real, spread-out +#: posting ages rather than a date that jumps to "now" on every re-crawl. +#: - `adzuna`: `created`, an aggregator's own record-creation stamp — named +#: as creation, not modification, like the rest. No live check: this +#: deployment carries no Adzuna key (`plugins.search`'s own docstring: +#: "not yet run live") and no Adzuna subscription exists to fetch. +#: +#: A host not listed here, or a role whose stored `published_at` fails the +#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the +#: safe default this migration is replacing everywhere else. +TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({ + "greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", + "personio", "jobindex", "adzuna", +}) + +#: A `published_at` before this is a parsing slip or a placeholder, not a +#: real posting date: no board here existed before 2015. +_POSTED_FLOOR = "2015-01-01T00:00:00+00:00" + + +def posted_host(raw: dict[str, Any]) -> str: + """The host a role's dates came from, read the way `ats` and `search` + both put it in `Item.raw`: `board` for an ATS board (`greenhouse`, + `lever`, ...), `provider` for a cross-company search (`jobindex`, + `adzuna`). '' when neither plugin wrote one.""" + return str(raw.get("board") or raw.get("provider") or "") + + +def posted_at(published_at: str, first_seen: str, host: str) -> str: + """The date a role counts as posted: the earlier of `first_seen` (when + the app itself first stored it) and `published_at` (what the board + claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and + `published_at` is neither absurdly old nor later than `first_seen` — a + role cannot be posted after the app already saw it, trusted host or not, + which also catches a `published_at` the board reports in the future + (`first_seen` is never later than the moment this runs). Both dates are + `Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings + is exact and needs no parsing. A role with no trusted date, or with one + that fails the check, keeps `first_seen`.""" + if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen: + return published_at + return first_seen + + def _json(value: Any, empty: str) -> Any: """A JSON text column as the Python value, tolerant like `JsonDict`: a value that is already parsed passes, and one that does not parse is the @@ -273,6 +346,10 @@ class Role(Model): feed_text: str = "" original_url: str = "" published_at: str + #: The date this role counts as posted: see `posted_at`. Used for + #: `recency` and the feed's newest order; `first_seen` stays what it + #: always was, when the app itself first stored the role. + posted_at: str first_seen: str last_seen: str closed_at: str | None = None @@ -445,12 +522,12 @@ class Agent(Model): class Candidate(Model): - """What scoring reads about one open role for one agent: when it was - first seen, its company's affinity for that agent, and per gate `1` - match, `0` no match, `-1` the column is empty.""" + """What scoring reads about one open role for one agent: its posted + date (for `recency`), its company's affinity for that agent, and per + gate `1` match, `0` no match, `-1` the column is empty.""" uid: str - first_seen: str + posted_at: str affinity: int = 0 gates: list[int] = [] diff --git a/app/static/roles.js b/app/static/roles.js index 2a816aa..8180ea9 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -509,7 +509,7 @@ ${role.company ? `${esc(role.company)}` : ""} ${role.place ? `${esc(role.place)}` : ""} ${badges(role)} - ${ago(role.first_seen)} + ${ago(role.posted_at)} ${role.gated_reason ? `${esc(role.gated_reason)}` : ""} diff --git a/app/storage/schema.py b/app/storage/schema.py index 75d7711..101197f 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -12,10 +12,13 @@ shipped. from __future__ import annotations +import json import logging import sqlite3 from collections.abc import Callable +from app.models import posted_at, posted_host + log = logging.getLogger(__name__) SCHEMA = """ @@ -56,9 +59,15 @@ CREATE TABLE IF NOT EXISTS roles ( feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over original_url TEXT NOT NULL DEFAULT '', -- the posting page published_at TEXT NOT NULL, -- what the source claims, UTC - -- What storing postings buys. first_seen is the only honest posted date, - -- since Greenhouse resets updated_at on every edit. closed_at is set when a - -- board fetched in full no longer lists the role, and cleared if it returns. + -- What storing postings buys. first_seen is when the app itself first + -- stored the role; posted_at (app.models.posted_at) is the earlier of + -- first_seen and published_at, trusted only for a host known to report a + -- creation date rather than one that moves on an edit + -- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order + -- read posted_at, first_seen stays "when the app found it". closed_at is + -- set when a board fetched in full no longer lists the role, and cleared + -- if it returns. + posted_at TEXT NOT NULL DEFAULT '', first_seen TEXT NOT NULL, last_seen TEXT NOT NULL, closed_at TEXT, @@ -241,6 +250,11 @@ CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key); CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key); CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC); CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at); +-- Serves the newest feed (open_only, ordered by posted_at DESC), which is +-- most of what `Store.roles` runs: partial, since only an open role's +-- position in that order is ever asked for. +CREATE INDEX IF NOT EXISTS idx_roles_posted + ON roles(posted_at DESC, uid) WHERE closed_at IS NULL; CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key); CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id); CREATE INDEX IF NOT EXISTS idx_application_events @@ -257,6 +271,23 @@ MARKER = "roles" Step = str | Callable[[sqlite3.Connection], None] + +def _migrate_posted_at(conn: sqlite3.Connection) -> None: + """v9's step: add `posted_at`, then backfill it for every row already + stored, by the same rule `Store.upsert_role` applies to a new one (see + `app.models.posted_at`). A callable, not SQL, because the rule reads + `raw` for which host a role came from. `'' NOT NULL` is a placeholder + only until the UPDATE below runs, in the same transaction as the ALTER + (`_apply` wraps every migration in one); every row gets a real value.""" + conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''") + rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall() + conn.executemany( + "UPDATE roles SET posted_at = ? WHERE uid = ?", + [(posted_at(row["published_at"], row["first_seen"], + posted_host(json.loads(row["raw"] or "{}"))), row["uid"]) + for row in rows]) + + MIGRATIONS: tuple[tuple[int, str, Step], ...] = ( # Never run as a migration: a file without the marker table is created # from `SCHEMA` and stamped. It exists so the first real migration is 2. @@ -338,6 +369,8 @@ ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT ''; (8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """ ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}'; """), + (9, "roles.posted_at: recency and the newest feed count from when a role " + "was posted, not just when the app found it", _migrate_posted_at), ) #: What a file created from `SCHEMA` today is already at. diff --git a/app/storage/store.py b/app/storage/store.py index c463971..238f363 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -23,7 +23,7 @@ from desk.models import Item from app.models import ( AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key, - country_of, place_key, remote_of, title_key, + country_of, place_key, posted_at, posted_host, remote_of, title_key, ) from app.storage import schema @@ -262,6 +262,12 @@ class Store: from `item.raw` under their own names, with `company` falling back to the subscription's and `country` to what `place` names; an empty value never overwrites a filled column. + + `posted_at` (see `app.models.posted_at`) is set from this pull's own + `published_at` and `stamp`; on an update the `ON CONFLICT` clause below + keeps the earlier of that and what is already stored, so an edit on + the board — which can only ever move `published_at` later, never + earlier — cannot move `posted_at` later either. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") @@ -269,6 +275,7 @@ class Store: company = str(raw.get("company") or (subscription.company if subscription else "")) place = str(raw.get("place") or "") stamp = self.now() + published = _iso(item.published_at) row: dict[str, Any] = { "uid": item.uid, "source": item.source, @@ -283,7 +290,8 @@ class Store: "description": item.body, "feed_text": item.excerpt, "original_url": item.location, - "published_at": _iso(item.published_at), + "published_at": published, + "posted_at": posted_at(published, stamp, posted_host(raw)), "first_seen": stamp, "last_seen": stamp, "raw": json.dumps(raw, default=str), @@ -301,6 +309,7 @@ class Store: for c in (*_RAW_INT, "subscription_id")] updates += [f"{c}=excluded.{c}" for c in ("title", "title_key", "published_at", "raw", "last_seen")] + updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)") updates.append("closed_at=NULL") columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) @@ -428,12 +437,12 @@ class Store: tags: Sequence[str] = (), tag_mode: str = "any", agent: int | None = None, by_score: bool = True, gated: bool = False, judged: str | None = None, offset: int = 0) -> list[ScoredRole]: - """Newest first by `first_seen`. `company` and `place` are + """Newest first by `posted_at`. `company` and `place` are case-insensitive substrings of the columns a board wrote — not the key, so a search does not have to know the exact slug a company folds to. `q` is matched word by word against `title`; a role must match every word. `function` and `remote` match a gate column exactly. - `days` keeps only roles first seen within that many days of now. + `days` keeps only roles posted within that many days of now. `agent` says whose votes, hidden companies and scores are meant. Each row carries that agent's `vote` and `voted_at`; `judged` (a key of @@ -470,10 +479,20 @@ class Store: tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated, judged=judged) select = "SELECT roles.*, v.vote, v.voted_at" - order = "roles.first_seen DESC, roles.uid" - if agent is not None and by_score: + order = "roles.posted_at DESC, roles.uid" + scored = agent is not None and by_score + if scored: select += f", {_VOTED_SCORE} AS score, s.gated_reason" order = "score DESC, " + order + elif open_only: + # Newest first, open only: idx_roles_posted covers this exactly + # (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the + # LIMIT rather than sorting every open role first. Left to the + # planner, sqlite3's bundled build (no STAT4) reads the near-100% + # selectivity of `closed_at IS NULL` from idx_roles_closed's plain + # average and picks that instead, sorting the lot — checked with + # EXPLAIN QUERY PLAN on a copy of the live database. + sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1) found = [ScoredRole.from_row(r) for r in self._rows( f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])] by_uid = self._tags_by_uid([r.uid for r in found]) @@ -520,7 +539,7 @@ class Store: where.append("remote = ?") params.append(remote) if days is not None: - where.append("first_seen >= ?") + where.append("posted_at >= ?") params.append(_iso(self.clock() - timedelta(days=days))) for word in (q or "").split(): where.append("title LIKE ? ESCAPE '\\'") @@ -640,7 +659,7 @@ class Store: return (f"""r.uid IN (SELECT uid FROM ( SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER ( PARTITION BY s.agent_id - ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n + ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n FROM scores s JOIN agents a ON a.id = s.agent_id AND a.enabled = 1 JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL @@ -935,7 +954,7 @@ class Store: columns.append( f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}") params += values - select = ", ".join(["r.uid", "r.first_seen", + select = ", ".join(["r.uid", "r.posted_at", "COALESCE(ac.affinity, 0) AS affinity", *columns]) rows = self._rows( f"""SELECT {select} FROM roles r @@ -943,7 +962,7 @@ class Store: ON ac.company_key = r.company_key AND ac.agent_id = ? WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""", [*params, agent_id]) - return [Candidate(uid=r["uid"], first_seen=r["first_seen"], + return [Candidate(uid=r["uid"], posted_at=r["posted_at"], affinity=int(r["affinity"]), gates=[int(r[f"g{n}"]) for n in range(len(gates))]) for r in rows] diff --git a/docs/decisions.md b/docs/decisions.md index d1a82d8..277c011 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -30,7 +30,9 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`permit` and `languages` are columns, and so is `extract_tries`.** `permit` answers the plan's open item 1 (visa sponsorship as a gate field) while nothing depends on it yet. newsdesk keeps give-up counts in `raw`; here `upsert_role` rewrites `raw` on every pull, which would reset the count, so the counter is its own column. -**Postings are persisted.** Storing every posting is what gives `first_seen`, which is the only honest posted date since Greenhouse resets `updated_at` on every edit, and `closed_at`, set by `mark_closed` when a complete fetch of a board no longer lists a role. A role listed again is open again. +**Postings are persisted.** Storing every posting is what gives `first_seen`, when the app itself first stored the role, and `closed_at`, set by `mark_closed` when a complete fetch of a board no longer lists a role. A role listed again is open again. + +**`posted_at` (migration 9) is the earlier of `first_seen` and a trusted host's `published_at`; `first_seen` is left alone, as when the app found the role.** `first_seen` being the only honest posted date was true of Greenhouse's `updated_at`, which resets on every edit; `plugins.ats` already reads `first_published` instead, and that plus the other hosts' dates — Lever's `createdAt`, Ashby's `publishedAt`, SmartRecruiters' `releasedDate`, Recruitee's `published_at`/`created_at`, Personio's `createdAt`, Workable's `published`, and the two `search` providers' (Jobindex's RSS `pubDate`, Adzuna's `created`) — are creation dates, checked against live boards and the live database (`app.models.TRUSTED_POSTED_HOSTS`, with the evidence in its comment). A date in the future or before 2015 is not trusted either, host or no host. Without this, a newly added board's whole backlog scored as freshly posted for two weeks: on the live database, 10,223 of 15,415 board roles were posted more than 30 days before they were first seen (average gap in days: Ashby 78, Greenhouse 143, Lever 226, Personio 333), and open roles with `recency` (`app.jobs.score.recency`) above 0.5 fell from all 15,558 of them to 3,089 (19.9%) once it read `posted_at` instead. `upsert_role` keeps the earliest `posted_at` a role has ever had: an edit on the board can only move its own date later, never earlier, so it can only move `posted_at` earlier too, when the board finally reports the true one. **The tracker schema is taken from jobhunt-hq (MIT).** `applications` with status in `saved, applied, screen, technical, onsite, offer, rejected, withdrawn`, `next_action`, `next_date`, and an append-only `application_events` written by the store on every status entered. The events table is what "days to first screen" and conversion by source are computed from. diff --git a/tests/test_score.py b/tests/test_score.py index 83daa34..85d63bb 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -80,9 +80,10 @@ def config(tmp_path: Path) -> Config: def role(store: Store, source_id: str, title: str = "Backend Engineer", *, - company: str = "Acme", body: str = "", **raw: Any) -> str: + company: str = "Acme", body: str = "", + published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: Any) -> str: item = Item(source_id=source_id, location=f"https://boards.example/{source_id}", - title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC), + title=title, body=body, published_at=published_at, raw={"company": company, **raw}) item.source = "ats" store.upsert_role(item) @@ -133,7 +134,7 @@ def test_recency_halves_every_fourteen_days() -> None: def test_a_role_without_a_vector_is_scored_without_fit_unless_fit_is_all() -> None: - candidate = Candidate(uid="u", first_seen=NOW.isoformat(), affinity=0) + candidate = Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0) value, reason = score.score_one(candidate, [], Weights(), None, NOW) assert reason == "" and value == pytest.approx((0.2 * 1.0 + 0.2 / 3) / 0.4) @@ -193,7 +194,7 @@ def test_failed_gates_reads_evidence_only() -> None: Gate(field="seniority", values=["senior"], mode="any"), Gate(field="function", values=["engineering"], mode="prefer")] def cand(*states: int) -> Candidate: - return Candidate(uid="u", first_seen=NOW.isoformat(), affinity=0, gates=list(states)) + return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0, gates=list(states)) assert score.failed_gates(cand(1, 1, 0, 0), gates) == [] assert score.failed_gates(cand(-1, -1, -1, 0), gates) == [] assert score.failed_gates(cand(0, 0, -1, 1), gates) == [ @@ -261,6 +262,22 @@ def test_an_older_role_scores_lower( assert scores[new][0] > scores[old][0] +def test_recency_counts_from_posted_at_not_first_seen( + store: Store, config: Config, fake: FakeEmbedder) -> None: + """A trusted board's backlog, first seen today, must not score as fresh: + recency has to read the board's own (trusted) posted date, not the + moment the app happened to pull it in.""" + stale = role(store, "1", board="greenhouse", + published_at=NOW - timedelta(days=28)) + fresh = role(store, "2") # untrusted host: posted_at falls back to first_seen (NOW) + agent = store.save_agent("Any") + + run(config, store, score.run_score) + + scores = store.scores(agent.id) + assert scores[fresh][0] > scores[stale][0] + + def test_two_agents_rank_the_same_roles_the_other_way_round( store: Store, config: Config, fake: FakeEmbedder) -> None: back = role(store, "1", "Backend Engineer", body="Python services") diff --git a/tests/test_storage.py b/tests/test_storage.py index c5e1675..740e626 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -11,7 +11,10 @@ import pytest from pydantic import ValidationError from desk.models import Item -from app.models import Application, Limits, company_key, place_key, title_key +from app.models import ( + Application, Limits, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at, + posted_host, title_key, +) from app.storage import LATEST, Store @@ -38,10 +41,11 @@ def store(tmp_path: Path, clock: Clock) -> Iterator[Store]: yield s -def posting(source_id: str = "123", **raw: object) -> Item: +def posting(source_id: str = "123", *, + published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: object) -> Item: item = Item(source_id=source_id, location=f"https://boards.example/acme/{source_id}", title="Senior Backend Engineer (m/w/d)", body="We build things.", - published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw)) + published_at=published_at, raw=dict(raw)) item.source = "ats" return item @@ -62,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 8 + assert LATEST == 9 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -86,8 +90,9 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None: Store(fresh) Store(old) with sqlite3.connect(old) as conn: + conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at for column in ("permit", "languages", "extract_tries", "summary", "summarised_at", - "article_text", "article_at", "read_at"): + "article_text", "article_at", "read_at", "posted_at"): conn.execute(f"ALTER TABLE roles DROP COLUMN {column}") for column in ("intent_key", "intent_vec", "limits"): # v3, v8 conn.execute(f"ALTER TABLE agents DROP COLUMN {column}") @@ -112,6 +117,8 @@ def _downgrade_to_v4(path: Path) -> None: the company, no read state, no tags, no per-agent tables.""" with sqlite3.connect(path) as conn: conn.execute("ALTER TABLE roles DROP COLUMN read_at") + conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at + conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9 conn.execute("DROP TABLE role_tags") conn.execute("ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1))") conn.execute("ALTER TABLE roles ADD COLUMN voted_at TEXT") @@ -205,6 +212,34 @@ def test_a_v4_file_migrates_read_state_and_tags(tmp_path: Path) -> None: assert "role_tags" in tables +def test_v9_backfills_posted_at_by_the_same_rule_upsert_uses( + tmp_path: Path, clock: Clock) -> None: + path = tmp_path / "v8.db" + store = Store(path) + store.clock = clock + old_date = datetime(2020, 1, 1, tzinfo=UTC) + trusted = posting("1", board="greenhouse", published_at=old_date) + untrusted = posting("2", published_at=old_date) + store.upsert_role(trusted) + store.upsert_role(untrusted) + before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid) + assert before_trusted is not None and before_untrusted is not None + + with sqlite3.connect(path) as conn: + conn.execute("DROP INDEX idx_roles_posted") + conn.execute("ALTER TABLE roles DROP COLUMN posted_at") + conn.execute("PRAGMA user_version = 8") + + migrated = Store(path) + + assert migrated.migrations_applied == [9] + after_trusted, after_untrusted = migrated.role(trusted.uid), migrated.role(untrusted.uid) + assert after_trusted is not None and after_untrusted is not None + # The backfill reproduces exactly what upsert_role would have written. + assert after_trusted.posted_at == before_trusted.posted_at == "2020-01-01T00:00:00+00:00" + assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen + + def test_state_is_a_mapping(store: Store) -> None: store.state["ats:last"] = "2026-09-23" assert store.state["ats:last"] == "2026-09-23" @@ -225,6 +260,37 @@ def test_keys() -> None: assert company_key("Celonis SE") == company_key("celonis") == "celonis" +# -------------------------------------------------------------- posted_at + +def test_posted_host_reads_board_then_provider() -> None: + assert posted_host({"board": "greenhouse", "provider": "jobindex"}) == "greenhouse" + assert posted_host({"provider": "jobindex"}) == "jobindex" + assert posted_host({}) == "" + + +def test_posted_at_trusts_only_a_known_host() -> None: + assert "greenhouse" in TRUSTED_POSTED_HOSTS and "workday" not in TRUSTED_POSTED_HOSTS + early = "2020-01-01T00:00:00+00:00" + late = "2026-09-23T09:00:00+00:00" + # A trusted host's earlier date wins over first_seen... + assert posted_at(early, late, "greenhouse") == early + # ...but an untrusted (or unknown) host never moves it off first_seen. + assert posted_at(early, late, "workday") == late + assert posted_at(early, late, "") == late + + +def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None: + first_seen = "2026-09-23T09:00:00+00:00" + future = "2026-09-24T00:00:00+00:00" # after first_seen + absurd = "2014-12-31T23:59:59+00:00" # before the 2015 floor + assert posted_at(future, first_seen, "greenhouse") == first_seen + assert posted_at(absurd, first_seen, "greenhouse") == first_seen + # The floor itself, and the instant first_seen, both still count. + assert posted_at("2015-01-01T00:00:00+00:00", first_seen, "greenhouse") == \ + "2015-01-01T00:00:00+00:00" + assert posted_at(first_seen, first_seen, "greenhouse") == first_seen + + # ------------------------------------------------------------------- roles def test_upsert_twice_keeps_first_seen_and_bumps_last_seen( @@ -259,6 +325,39 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000) +def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest( + store: Store, clock: Clock) -> None: + """A board's first pull can carry a stale, too-recent published_at (a + board that has not backfilled first_published yet, say); a later pull + that finally reports the true, earlier date must still win — an edit on + the board only ever moves published_at later, never earlier, so the + earliest posted_at ever computed is the one to keep.""" + store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20, + tzinfo=UTC))) + first = store.role(posting().uid) + assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00" + + clock.tick() + store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1, + tzinfo=UTC))) + earlier = store.role(posting().uid) + assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00" + + clock.tick() + # A later, LATER published_at (an edit) must not push posted_at forward. + store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25, + tzinfo=UTC))) + still_earlier = store.role(posting().uid) + assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00" + + +def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None: + store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC))) + role = store.role(posting().uid) + assert role is not None + assert role.posted_at == role.first_seen + + def test_company_falls_back_to_the_subscription(store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse") store.upsert_role(posting(), sub) @@ -279,6 +378,58 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None: assert [r.source_id for r in store.roles(limit=1)] == ["3"] +def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None: + """A role from a trusted board found today but posted months ago must not + outrank one the app has genuinely known about for longer.""" + store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen + clock.tick() + store.upsert_role(posting("old", board="greenhouse", + published_at=datetime(2020, 1, 1, tzinfo=UTC))) + assert [r.source_id for r in store.roles()] == ["recent", "old"] + assert [r.source_id for r in store.roles(days=30)] == ["recent"] + + +def _spy_sql(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Every SQL statement a `Store` runs through `_rows`, in order.""" + captured: list[str] = [] + original = Store._rows + + def spy(self: Store, sql: str, params: object = ()) -> list[sqlite3.Row]: + captured.append(sql) + return original(self, sql, params) + + monkeypatch.setattr(Store, "_rows", spy) + return captured + + +def test_the_newest_feed_query_uses_the_partial_posted_at_index( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + """`Store.roles`' default order (newest, open only) is forced onto + `idx_roles_posted`: left to the planner's plain, non-STAT4 statistics, it + can instead pick `idx_roles_closed` and sort every open role, which on + the live database (15,558 of 15,815 roles open) measured over 30x slower + than the plan this asserts (see docs/decisions.md).""" + store.upsert_role(posting("1")) + sql, _ = store._roles_from( + company=None, place=None, open_only=True, q=None, function=None, remote=None, + days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any", + agent=None, by_score=True, gated=False, judged=None) + assert "INDEXED BY" not in sql # the hint belongs to roles(), not the shared builder + + captured = _spy_sql(monkeypatch) + store.roles(limit=10) + assert "INDEXED BY idx_roles_posted" in captured[0] + with sqlite3.connect(store.path) as conn: + plan = " ".join(row[3] for row in + conn.execute("EXPLAIN QUERY PLAN " + captured[0], [None, 10, 0])) + assert "USING INDEX idx_roles_posted" in plan and "TEMP B-TREE" not in plan + + # open_only=False cannot use it: the index is partial on closed_at IS NULL. + captured.clear() + store.roles(limit=10, open_only=False) + assert "INDEXED BY" not in captured[0] + + def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None: store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering", remote=True))