Serve a row projection from GET /api/roles instead of the whole role
The feed only ever shows a title, a few badges and a short excerpt, but the list endpoint answered with every role's full row -- description, feed_text, raw and article_text included, about 5.5 KB a role -- so a 1000-row restore was several megabytes of text a phone never renders. Store.roles_page selects only the fields a row draws (RoleRow), with excerpt built server-side off a SUBSTR of summary or description, so the big columns never cross into the response. The reader is unaffected: it already fetched the whole role by uid when it opened one; the reader pane's own reliance on the list row carrying the body is removed too, so it no longer flashes a wrong "no description" before that fetch lands. Measured on a copy of the live database: a 60-row page went from 392 KiB to 39 KiB, and a 1000-row restore from 6.6 MB to 631 KiB. Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
1 parent
e6be593542
commit
91596300e1
6 files changed
+251
-49
No files matched your search
+57
-2
@@ -388,13 +388,68 @@ class ExtractResult(ModelTaskResult):
|
||||
|
||||
|
||||
class ScoredRole(Role):
|
||||
"""A role as `/api/roles` serves it: with one agent's score and gate
|
||||
reason when an agent was asked for, and without otherwise."""
|
||||
"""A role as `/api/roles/{uid}` serves it: with one agent's score and
|
||||
gate reason when an agent was asked for, and without otherwise."""
|
||||
|
||||
score: float | None = None
|
||||
gated_reason: str = ""
|
||||
|
||||
|
||||
#: How many characters `RoleRow.excerpt` keeps at most: the feed row's own
|
||||
#: cap, built once here rather than by the client, so a list response never
|
||||
#: carries more of a role's text than a row draws.
|
||||
EXCERPT_LIMIT = 200
|
||||
|
||||
|
||||
def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str:
|
||||
"""`text` (a role's summary, or the opening of its description) with its
|
||||
whitespace collapsed, cut to `limit` characters at a word boundary with
|
||||
an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a
|
||||
`SUBSTR` of the source column rather than the whole of it, so the feed's
|
||||
list never carries what only the reader needs."""
|
||||
collapsed = " ".join((text or "").split())
|
||||
if len(collapsed) <= limit:
|
||||
return collapsed
|
||||
cut = collapsed[:limit]
|
||||
space = cut.rfind(" ")
|
||||
trimmed = cut[:space] if space > limit * 0.6 else cut
|
||||
return trimmed.rstrip(" ,.;:" + chr(0x2013) + "-") + "…"
|
||||
|
||||
|
||||
class RoleRow(Model):
|
||||
"""One row of the feed: what `GET /api/roles` serves per role, the
|
||||
fields a row draws and nothing else — never `description`, `feed_text`,
|
||||
`raw` or `article_text`, the four columns docs/decisions.md measures at
|
||||
about 5.5 KB a role between them. The reader fetches the rest by uid when
|
||||
it opens a role (`GET /api/roles/{uid}`, the full `Role`).
|
||||
|
||||
`excerpt` is built on the server (see `excerpt`, above): the summary when
|
||||
there is one, else the description's opening, cut to `EXCERPT_LIMIT`
|
||||
characters. `salary_min`, `salary_max` and `currency` are the pay fields
|
||||
a row already draws; `salary_text` and `salary_state`, once the role
|
||||
model carries them, belong here too."""
|
||||
|
||||
uid: str
|
||||
company_key: str = ""
|
||||
company: str = ""
|
||||
title: str = ""
|
||||
place: str = ""
|
||||
country: str = ""
|
||||
remote: Remote = ""
|
||||
function: str = ""
|
||||
salary_min: int | None = None
|
||||
salary_max: int | None = None
|
||||
currency: str = ""
|
||||
posted_at: str = ""
|
||||
score: float | None = None
|
||||
gated_reason: str = ""
|
||||
vote: Literal[-1, 0, 1] | None = None
|
||||
voted_at: str | None = None
|
||||
read_at: str | None = None
|
||||
tags: list[str] = []
|
||||
excerpt: str = ""
|
||||
|
||||
|
||||
class Company(Model):
|
||||
"""A row of `companies`, with `affinity` as the agent it was read for
|
||||
has it (see `agent_companies`); 0 when no agent was asked for."""
|
||||
|
||||
+26
-14
@@ -473,15 +473,23 @@
|
||||
for (const el of had.values()) el.remove();
|
||||
}
|
||||
|
||||
/* The summary when there is one, else the description's opening, cut at a
|
||||
word. */
|
||||
/* Built on the server (`app.models.excerpt`) and served as `role.excerpt`,
|
||||
so the list response never carries the full description a row would
|
||||
otherwise need to cut this from — see docs/decisions.md, "The pages". */
|
||||
function excerpt(role) {
|
||||
if (role.summary) return role.summary;
|
||||
const text = (role.description || "").replace(/\s+/g, " ").trim();
|
||||
if (text.length <= 160) return text;
|
||||
const cut = text.slice(0, 160);
|
||||
return role.excerpt || "";
|
||||
}
|
||||
|
||||
/* The same cut, run here instead: for the one row `merge` refreshes after
|
||||
a fetch or a summarise, off the full role that call answers with, which
|
||||
is the only time this page ever sees a role's `summary` or
|
||||
`description` directly. */
|
||||
function excerptOf(role) {
|
||||
const text = (role.summary || role.description || "").replace(/\s+/g, " ").trim();
|
||||
if (text.length <= 200) return text;
|
||||
const cut = text.slice(0, 200);
|
||||
const space = cut.lastIndexOf(" ");
|
||||
return `${(space > 100 ? cut.slice(0, space) : cut).replace(/[\s,.;:–-]+$/, "")}…`;
|
||||
return `${(space > 120 ? cut.slice(0, space) : cut).replace(/[\s,.;:–-]+$/, "")}…`;
|
||||
}
|
||||
|
||||
function tagChips(tags) {
|
||||
@@ -723,9 +731,13 @@
|
||||
}
|
||||
if (!open) { el.replaceChildren(); return; }
|
||||
|
||||
// The list row is already in memory, and a list row is the whole role, so
|
||||
// the pane paints from it at once and the fetch only brings it up to date.
|
||||
// The list row is already in memory, and carries what a row draws —
|
||||
// title, company, place, tags, pay — so the pane paints those at once;
|
||||
// the body, the summary state and the original link wait on the fetch
|
||||
// (`loaded`), since the list row no longer carries them (see
|
||||
// `Store.roles_page`).
|
||||
const stub = state.items.find((r) => r.uid === state.readingUid);
|
||||
const loaded = Boolean(state.reading) && state.reading.uid === state.readingUid;
|
||||
const r = state.reading || stub || {};
|
||||
const same = el.dataset.uid === state.readingUid;
|
||||
const scrolled = same ? el.querySelector(".reader-body")?.scrollTop || 0 : 0;
|
||||
@@ -736,13 +748,13 @@
|
||||
const text = (r.article_text || r.description || "").trim();
|
||||
const body = error
|
||||
? `<p class="reader-note">This role could not be loaded: ${esc(error)}</p>`
|
||||
: !r.uid
|
||||
: !loaded
|
||||
? `<p class="reader-note">Loading…</p>`
|
||||
: text ? paragraphs(text)
|
||||
: `<p class="reader-note">The board gave no description for this one.</p>`;
|
||||
const pay = salary(r);
|
||||
const waiting = state.summarising === state.readingUid;
|
||||
const teaser = r.uid && !r.article_text && out && (r.description || "").length < TEASER;
|
||||
const teaser = loaded && !r.article_text && out && (r.description || "").length < TEASER;
|
||||
|
||||
el.innerHTML = `
|
||||
<div class="reader-bar">
|
||||
@@ -767,7 +779,7 @@
|
||||
${r.summary ? `<section class="reader-summary">
|
||||
<h2 class="reader-label">Summary</h2><p>${esc(r.summary)}</p></section>` : ""}
|
||||
<div class="reader-tools">
|
||||
${r.uid && !r.summary && !waiting
|
||||
${loaded && !r.summary && !waiting
|
||||
? `<button type="button" class="secondary" data-tool="summarise">Summarise</button>` : ""}
|
||||
${teaser ? `<button type="button" class="secondary" data-tool="fetch">Fetch the posting</button>` : ""}
|
||||
<span class="muted small" role="status">${waiting
|
||||
@@ -819,9 +831,9 @@
|
||||
else state.reading = updated;
|
||||
const item = state.items.find((x) => x.uid === updated.uid);
|
||||
if (item) {
|
||||
Object.assign(item, { summary: updated.summary, article_text: updated.article_text });
|
||||
item.excerpt = excerptOf(updated);
|
||||
const row = $(`#roles-list .row[data-uid="${CSS.escape(updated.uid)}"]`);
|
||||
if (row) row.role = null; // drawn again from the new text
|
||||
if (row) row.role = null; // drawn again from the new excerpt
|
||||
renderFeed();
|
||||
}
|
||||
renderReader();
|
||||
|
||||
+63
-2
@@ -22,8 +22,8 @@ import numpy as np
|
||||
from desk.models import Item
|
||||
from app.models import (
|
||||
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
||||
Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key,
|
||||
country_of, place_key, posted_at, posted_host, remote_of, title_key,
|
||||
Company, Gate, Limits, Role, RoleRow, ScoredRole, Subscription, Weights, company_key,
|
||||
country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.storage import schema
|
||||
|
||||
@@ -70,6 +70,17 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
|
||||
#: nothing, so `vote` and `voted_at` are NULL: one query shape either way.
|
||||
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
||||
|
||||
#: What `roles_page` reads for `RoleRow.excerpt`: the summary when there is
|
||||
#: one, else the description's opening. A `SUBSTR` of the source column, not
|
||||
#: the column itself, so a big `description` never leaves SQLite whole for a
|
||||
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
|
||||
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
|
||||
|
||||
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
|
||||
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
|
||||
#: still has somewhere to land.
|
||||
_EXCERPT_RAW = 240
|
||||
|
||||
_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city",
|
||||
"ats_slug")
|
||||
|
||||
@@ -521,6 +532,56 @@ class Store:
|
||||
r.tags = by_uid.get(r.uid, [])
|
||||
return found
|
||||
|
||||
def roles_page(self, limit: int = 100, company: str | None = None,
|
||||
place: str | None = None, open_only: bool = True,
|
||||
q: str | None = None, function: str | None = None,
|
||||
remote: str | None = None, days: int | None = None,
|
||||
hide_muted: bool = False, include_read: bool = True,
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, offset: int = 0) -> list[RoleRow]:
|
||||
"""What `GET /api/roles` serves: the same rows `roles` would, in the
|
||||
same order, with the same filters and paging, but only the fields a
|
||||
row draws (see `RoleRow`) — never `description`, `feed_text`, `raw`
|
||||
or `article_text`, which a full `roles.*` select would carry whether
|
||||
or not a page ever renders them. The reader still gets the whole role
|
||||
by uid, see `role`.
|
||||
|
||||
`excerpt` is `app.models.excerpt` over a `SUBSTR` of `_EXCERPT_SOURCE`
|
||||
(`_EXCERPT_RAW` characters, trimmed further in Python), so the source
|
||||
column itself never crosses out of SQLite for a list."""
|
||||
sql, params = self._roles_from(
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged)
|
||||
select = (
|
||||
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
|
||||
"roles.country, roles.remote, roles.function, roles.salary_min, "
|
||||
"roles.salary_max, roles.currency, roles.posted_at, roles.read_at, "
|
||||
"v.vote, v.voted_at, "
|
||||
f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt")
|
||||
order = "roles.posted_at DESC, roles.uid"
|
||||
scored = agent is not None and by_score
|
||||
if scored:
|
||||
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
|
||||
order = "score DESC, " + order
|
||||
elif open_only:
|
||||
# Same reasoning as `roles`: the planner otherwise sorts every
|
||||
# open role before applying LIMIT.
|
||||
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
|
||||
rows = self._rows(f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?",
|
||||
[*params, limit, offset])
|
||||
found = []
|
||||
for r in rows:
|
||||
row = dict(r)
|
||||
row["excerpt"] = excerpt(row["excerpt"])
|
||||
found.append(RoleRow.from_row(row))
|
||||
by_uid = self._tags_by_uid([r.uid for r in found])
|
||||
for r in found:
|
||||
r.tags = by_uid.get(r.uid, [])
|
||||
return found
|
||||
|
||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||
open_only: bool = True, q: str | None = None,
|
||||
function: str | None = None, remote: str | None = None,
|
||||
|
||||
+9
-4
@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app import jobs
|
||||
from app.models import (
|
||||
AFFINITY, Agent, Application, Company, Gate, Limits, Role, ScoredRole, Status,
|
||||
AFFINITY, Agent, Application, Company, Gate, Limits, Role, RoleRow, Status,
|
||||
Subscription, Weights, company_key,
|
||||
)
|
||||
from app.storage import Store
|
||||
@@ -111,9 +111,14 @@ def unread_role(uid: str, store: StoreDep) -> Role:
|
||||
|
||||
|
||||
class RolesPage(BaseModel):
|
||||
"""One page of roles, and how many the same filters match in all."""
|
||||
"""One page of roles, and how many the same filters match in all. `items`
|
||||
is `RoleRow`, not the full `Role`: the fields a feed row draws, with an
|
||||
excerpt built on the server, never the big columns (`description`,
|
||||
`feed_text`, `raw`, `article_text`) a full role carries — see
|
||||
`Store.roles_page`. The reader fetches the rest by uid when it opens one,
|
||||
`GET /api/roles/{uid}`."""
|
||||
|
||||
items: list[ScoredRole]
|
||||
items: list[RoleRow]
|
||||
total: int
|
||||
|
||||
|
||||
@@ -161,7 +166,7 @@ def list_roles(
|
||||
"hide_muted": not show_muted, "include_read": include_read, "tags": tags,
|
||||
"tag_mode": tag_mode, "agent": agent_id, "by_score": by_score,
|
||||
"gated": gated, "judged": judged or None}
|
||||
return RolesPage(items=store.roles(limit=limit, offset=offset, **filters),
|
||||
return RolesPage(items=store.roles_page(limit=limit, offset=offset, **filters),
|
||||
total=store.count_roles(**filters))
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user