Serve a row projection from GET /api/roles instead of the whole role

The feed only ever shows a title, a few badges and a short excerpt,
but the list endpoint answered with every role's full row --
description, feed_text, raw and article_text included, about 5.5 KB
a role -- so a 1000-row restore was several megabytes of text a phone
never renders.

Store.roles_page selects only the fields a row draws (RoleRow), with
excerpt built server-side off a SUBSTR of summary or description, so
the big columns never cross into the response. The reader is
unaffected: it already fetched the whole role by uid when it opened
one; the reader pane's own reliance on the list row carrying the body
is removed too, so it no longer flashes a wrong "no description"
before that fetch lands.

Measured on a copy of the live database: a 60-row page went from
392 KiB to 39 KiB, and a 1000-row restore from 6.6 MB to 631 KiB.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet committed 2026-09-29 15:22:00 +02:00
1 parent e6be593542
commit 91596300e1
6 files changed
+251 -49

No files matched your search

+63 -2
View File
@@ -22,8 +22,8 @@ import numpy as np
from desk.models import Item
from app.models import (
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key,
country_of, place_key, posted_at, posted_host, remote_of, title_key,
Company, Gate, Limits, Role, RoleRow, ScoredRole, Subscription, Weights, company_key,
country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key,
)
from app.storage import schema
@@ -70,6 +70,17 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
#: nothing, so `vote` and `voted_at` are NULL: one query shape either way.
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
#: What `roles_page` reads for `RoleRow.excerpt`: the summary when there is
#: one, else the description's opening. A `SUBSTR` of the source column, not
#: the column itself, so a big `description` never leaves SQLite whole for a
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
#: still has somewhere to land.
_EXCERPT_RAW = 240
_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city",
"ats_slug")
@@ -521,6 +532,56 @@ class Store:
r.tags = by_uid.get(r.uid, [])
return found
def roles_page(self, limit: int = 100, company: str | None = None,
place: str | None = None, open_only: bool = True,
q: str | None = None, function: str | None = None,
remote: str | None = None, days: int | None = None,
hide_muted: bool = False, include_read: bool = True,
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, offset: int = 0) -> list[RoleRow]:
"""What `GET /api/roles` serves: the same rows `roles` would, in the
same order, with the same filters and paging, but only the fields a
row draws (see `RoleRow`) — never `description`, `feed_text`, `raw`
or `article_text`, which a full `roles.*` select would carry whether
or not a page ever renders them. The reader still gets the whole role
by uid, see `role`.
`excerpt` is `app.models.excerpt` over a `SUBSTR` of `_EXCERPT_SOURCE`
(`_EXCERPT_RAW` characters, trimmed further in Python), so the source
column itself never crosses out of SQLite for a list."""
sql, params = self._roles_from(
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
select = (
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
"roles.country, roles.remote, roles.function, roles.salary_min, "
"roles.salary_max, roles.currency, roles.posted_at, roles.read_at, "
"v.vote, v.voted_at, "
f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt")
order = "roles.posted_at DESC, roles.uid"
scored = agent is not None and by_score
if scored:
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
order = "score DESC, " + order
elif open_only:
# Same reasoning as `roles`: the planner otherwise sorts every
# open role before applying LIMIT.
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
rows = self._rows(f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?",
[*params, limit, offset])
found = []
for r in rows:
row = dict(r)
row["excerpt"] = excerpt(row["excerpt"])
found.append(RoleRow.from_row(row))
by_uid = self._tags_by_uid([r.uid for r in found])
for r in found:
r.tags = by_uid.get(r.uid, [])
return found
def count_roles(self, company: str | None = None, place: str | None = None,
open_only: bool = True, q: str | None = None,
function: str | None = None, remote: str | None = None,