Merge branch 'worktree-agent-a3bd292551395dda0'
# Conflicts: # app/storage/schema.py
This commit is contained in:
commit
43773efc53
8 files changed
+338
-34
No files matched your search
+159
-6
@@ -11,7 +11,10 @@ import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from desk.models import Item
|
||||
from app.models import Application, Limits, company_key, place_key, title_key
|
||||
from app.models import (
|
||||
Application, Limits, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at,
|
||||
posted_host, title_key,
|
||||
)
|
||||
from app.storage import LATEST, Store
|
||||
|
||||
|
||||
@@ -38,10 +41,11 @@ def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
|
||||
yield s
|
||||
|
||||
|
||||
def posting(source_id: str = "123", **raw: object) -> Item:
|
||||
def posting(source_id: str = "123", *,
|
||||
published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: object) -> Item:
|
||||
item = Item(source_id=source_id, location=f"https://boards.example/acme/{source_id}",
|
||||
title="Senior Backend Engineer (m/w/d)", body="We build things.",
|
||||
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
|
||||
published_at=published_at, raw=dict(raw))
|
||||
item.source = "ats"
|
||||
return item
|
||||
|
||||
@@ -62,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 9
|
||||
assert LATEST == 10
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -86,8 +90,9 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
|
||||
Store(fresh)
|
||||
Store(old)
|
||||
with sqlite3.connect(old) as conn:
|
||||
conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
|
||||
for column in ("permit", "languages", "extract_tries", "summary", "summarised_at",
|
||||
"article_text", "article_at", "read_at"):
|
||||
"article_text", "article_at", "read_at", "posted_at"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
for column in ("intent_key", "intent_vec", "limits"): # v3, v8
|
||||
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
|
||||
@@ -112,11 +117,13 @@ def test_a_v8_file_gains_ats_slug_and_keeps_its_companies(tmp_path: Path) -> Non
|
||||
Store(path).add_company("Acme", ats="teamtailor", country="DK")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("ALTER TABLE companies DROP COLUMN ats_slug")
|
||||
conn.execute("DROP INDEX idx_roles_posted") # v10, which follows
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
|
||||
conn.execute("PRAGMA user_version = 8")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [9]
|
||||
assert migrated.migrations_applied == [9, 10]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.country, acme.ats_slug) == ("teamtailor", "DK", "")
|
||||
@@ -127,6 +134,8 @@ def _downgrade_to_v4(path: Path) -> None:
|
||||
the company, no read state, no tags, no per-agent tables."""
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN read_at")
|
||||
conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9
|
||||
conn.execute("DROP TABLE role_tags")
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1))")
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN voted_at TEXT")
|
||||
@@ -220,6 +229,34 @@ def test_a_v4_file_migrates_read_state_and_tags(tmp_path: Path) -> None:
|
||||
assert "role_tags" in tables
|
||||
|
||||
|
||||
def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
|
||||
tmp_path: Path, clock: Clock) -> None:
|
||||
path = tmp_path / "v9.db"
|
||||
store = Store(path)
|
||||
store.clock = clock
|
||||
old_date = datetime(2020, 1, 1, tzinfo=UTC)
|
||||
trusted = posting("1", board="greenhouse", published_at=old_date)
|
||||
untrusted = posting("2", published_at=old_date)
|
||||
store.upsert_role(trusted)
|
||||
store.upsert_role(untrusted)
|
||||
before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid)
|
||||
assert before_trusted is not None and before_untrusted is not None
|
||||
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("DROP INDEX idx_roles_posted")
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
|
||||
conn.execute("PRAGMA user_version = 9")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [10]
|
||||
after_trusted, after_untrusted = migrated.role(trusted.uid), migrated.role(untrusted.uid)
|
||||
assert after_trusted is not None and after_untrusted is not None
|
||||
# The backfill reproduces exactly what upsert_role would have written.
|
||||
assert after_trusted.posted_at == before_trusted.posted_at == "2020-01-01T00:00:00+00:00"
|
||||
assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen
|
||||
|
||||
|
||||
def test_state_is_a_mapping(store: Store) -> None:
|
||||
store.state["ats:last"] = "2026-09-23"
|
||||
assert store.state["ats:last"] == "2026-09-23"
|
||||
@@ -240,6 +277,37 @@ def test_keys() -> None:
|
||||
assert company_key("Celonis SE") == company_key("celonis") == "celonis"
|
||||
|
||||
|
||||
# -------------------------------------------------------------- posted_at
|
||||
|
||||
def test_posted_host_reads_board_then_provider() -> None:
|
||||
assert posted_host({"board": "greenhouse", "provider": "jobindex"}) == "greenhouse"
|
||||
assert posted_host({"provider": "jobindex"}) == "jobindex"
|
||||
assert posted_host({}) == ""
|
||||
|
||||
|
||||
def test_posted_at_trusts_only_a_known_host() -> None:
|
||||
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "workday" not in TRUSTED_POSTED_HOSTS
|
||||
early = "2020-01-01T00:00:00+00:00"
|
||||
late = "2026-09-23T09:00:00+00:00"
|
||||
# A trusted host's earlier date wins over first_seen...
|
||||
assert posted_at(early, late, "greenhouse") == early
|
||||
# ...but an untrusted (or unknown) host never moves it off first_seen.
|
||||
assert posted_at(early, late, "workday") == late
|
||||
assert posted_at(early, late, "") == late
|
||||
|
||||
|
||||
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
|
||||
first_seen = "2026-09-23T09:00:00+00:00"
|
||||
future = "2026-09-24T00:00:00+00:00" # after first_seen
|
||||
absurd = "2014-12-31T23:59:59+00:00" # before the 2015 floor
|
||||
assert posted_at(future, first_seen, "greenhouse") == first_seen
|
||||
assert posted_at(absurd, first_seen, "greenhouse") == first_seen
|
||||
# The floor itself, and the instant first_seen, both still count.
|
||||
assert posted_at("2015-01-01T00:00:00+00:00", first_seen, "greenhouse") == \
|
||||
"2015-01-01T00:00:00+00:00"
|
||||
assert posted_at(first_seen, first_seen, "greenhouse") == first_seen
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- roles
|
||||
|
||||
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
|
||||
@@ -274,6 +342,39 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
||||
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
|
||||
|
||||
|
||||
def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest(
|
||||
store: Store, clock: Clock) -> None:
|
||||
"""A board's first pull can carry a stale, too-recent published_at (a
|
||||
board that has not backfilled first_published yet, say); a later pull
|
||||
that finally reports the true, earlier date must still win — an edit on
|
||||
the board only ever moves published_at later, never earlier, so the
|
||||
earliest posted_at ever computed is the one to keep."""
|
||||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20,
|
||||
tzinfo=UTC)))
|
||||
first = store.role(posting().uid)
|
||||
assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00"
|
||||
|
||||
clock.tick()
|
||||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1,
|
||||
tzinfo=UTC)))
|
||||
earlier = store.role(posting().uid)
|
||||
assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00"
|
||||
|
||||
clock.tick()
|
||||
# A later, LATER published_at (an edit) must not push posted_at forward.
|
||||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25,
|
||||
tzinfo=UTC)))
|
||||
still_earlier = store.role(posting().uid)
|
||||
assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00"
|
||||
|
||||
|
||||
def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None:
|
||||
store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
|
||||
role = store.role(posting().uid)
|
||||
assert role is not None
|
||||
assert role.posted_at == role.first_seen
|
||||
|
||||
|
||||
def test_company_falls_back_to_the_subscription(store: Store) -> None:
|
||||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
|
||||
store.upsert_role(posting(), sub)
|
||||
@@ -294,6 +395,58 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
|
||||
assert [r.source_id for r in store.roles(limit=1)] == ["3"]
|
||||
|
||||
|
||||
def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None:
|
||||
"""A role from a trusted board found today but posted months ago must not
|
||||
outrank one the app has genuinely known about for longer."""
|
||||
store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen
|
||||
clock.tick()
|
||||
store.upsert_role(posting("old", board="greenhouse",
|
||||
published_at=datetime(2020, 1, 1, tzinfo=UTC)))
|
||||
assert [r.source_id for r in store.roles()] == ["recent", "old"]
|
||||
assert [r.source_id for r in store.roles(days=30)] == ["recent"]
|
||||
|
||||
|
||||
def _spy_sql(monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||||
"""Every SQL statement a `Store` runs through `_rows`, in order."""
|
||||
captured: list[str] = []
|
||||
original = Store._rows
|
||||
|
||||
def spy(self: Store, sql: str, params: object = ()) -> list[sqlite3.Row]:
|
||||
captured.append(sql)
|
||||
return original(self, sql, params)
|
||||
|
||||
monkeypatch.setattr(Store, "_rows", spy)
|
||||
return captured
|
||||
|
||||
|
||||
def test_the_newest_feed_query_uses_the_partial_posted_at_index(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`Store.roles`' default order (newest, open only) is forced onto
|
||||
`idx_roles_posted`: left to the planner's plain, non-STAT4 statistics, it
|
||||
can instead pick `idx_roles_closed` and sort every open role, which on
|
||||
the live database (15,558 of 15,815 roles open) measured over 30x slower
|
||||
than the plan this asserts (see docs/decisions.md)."""
|
||||
store.upsert_role(posting("1"))
|
||||
sql, _ = store._roles_from(
|
||||
company=None, place=None, open_only=True, q=None, function=None, remote=None,
|
||||
days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any",
|
||||
agent=None, by_score=True, gated=False, judged=None)
|
||||
assert "INDEXED BY" not in sql # the hint belongs to roles(), not the shared builder
|
||||
|
||||
captured = _spy_sql(monkeypatch)
|
||||
store.roles(limit=10)
|
||||
assert "INDEXED BY idx_roles_posted" in captured[0]
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
plan = " ".join(row[3] for row in
|
||||
conn.execute("EXPLAIN QUERY PLAN " + captured[0], [None, 10, 0]))
|
||||
assert "USING INDEX idx_roles_posted" in plan and "TEMP B-TREE" not in plan
|
||||
|
||||
# open_only=False cannot use it: the index is partial on closed_at IS NULL.
|
||||
captured.clear()
|
||||
store.roles(limit=10, open_only=False)
|
||||
assert "INDEXED BY" not in captured[0]
|
||||
|
||||
|
||||
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
|
||||
store.upsert_role(posting("1", company="Acme", place="Berlin",
|
||||
function="engineering", remote=True))
|
||||
|
||||
Reference in new issue
Block a user