Merge branch 'worktree-agent-a3bd292551395dda0'
# Conflicts: # app/storage/schema.py
This commit is contained in:
commit
43773efc53
8 files changed
+338
-34
No files matched your search
+36
-3
@@ -12,10 +12,13 @@ shipped.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app.models import posted_at, posted_host
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = """
|
||||
@@ -56,9 +59,15 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
|
||||
original_url TEXT NOT NULL DEFAULT '', -- the posting page
|
||||
published_at TEXT NOT NULL, -- what the source claims, UTC
|
||||
-- What storing postings buys. first_seen is the only honest posted date,
|
||||
-- since Greenhouse resets updated_at on every edit. closed_at is set when a
|
||||
-- board fetched in full no longer lists the role, and cleared if it returns.
|
||||
-- What storing postings buys. first_seen is when the app itself first
|
||||
-- stored the role; posted_at (app.models.posted_at) is the earlier of
|
||||
-- first_seen and published_at, trusted only for a host known to report a
|
||||
-- creation date rather than one that moves on an edit
|
||||
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
|
||||
-- read posted_at, first_seen stays "when the app found it". closed_at is
|
||||
-- set when a board fetched in full no longer lists the role, and cleared
|
||||
-- if it returns.
|
||||
posted_at TEXT NOT NULL DEFAULT '',
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
closed_at TEXT,
|
||||
@@ -244,6 +253,11 @@ CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
|
||||
-- Serves the newest feed (open_only, ordered by posted_at DESC), which is
|
||||
-- most of what `Store.roles` runs: partial, since only an open role's
|
||||
-- position in that order is ever asked for.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_posted
|
||||
ON roles(posted_at DESC, uid) WHERE closed_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_application_events
|
||||
@@ -260,6 +274,23 @@ MARKER = "roles"
|
||||
|
||||
Step = str | Callable[[sqlite3.Connection], None]
|
||||
|
||||
|
||||
def _migrate_posted_at(conn: sqlite3.Connection) -> None:
|
||||
"""v10's step: add `posted_at`, then backfill it for every row already
|
||||
stored, by the same rule `Store.upsert_role` applies to a new one (see
|
||||
`app.models.posted_at`). A callable, not SQL, because the rule reads
|
||||
`raw` for which host a role came from. `'' NOT NULL` is a placeholder
|
||||
only until the UPDATE below runs, in the same transaction as the ALTER
|
||||
(`_apply` wraps every migration in one); every row gets a real value."""
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''")
|
||||
rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall()
|
||||
conn.executemany(
|
||||
"UPDATE roles SET posted_at = ? WHERE uid = ?",
|
||||
[(posted_at(row["published_at"], row["first_seen"],
|
||||
posted_host(json.loads(row["raw"] or "{}"))), row["uid"])
|
||||
for row in rows])
|
||||
|
||||
|
||||
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
||||
# Never run as a migration: a file without the marker table is created
|
||||
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
||||
@@ -344,6 +375,8 @@ ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}';
|
||||
(9, "companies.ats_slug: the slug of a host the app cannot read yet is kept", """
|
||||
ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
(10, "roles.posted_at: recency and the newest feed count from when a role "
|
||||
"was posted, not just when the app found it", _migrate_posted_at),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
|
||||
Reference in new issue
Block a user