From 91596300e19b1a4f526c853a664ccbdd6f1de366 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 15:22:00 +0200 Subject: [PATCH] Serve a row projection from GET /api/roles instead of the whole role The feed only ever shows a title, a few badges and a short excerpt, but the list endpoint answered with every role's full row -- description, feed_text, raw and article_text included, about 5.5 KB a role -- so a 1000-row restore was several megabytes of text a phone never renders. Store.roles_page selects only the fields a row draws (RoleRow), with excerpt built server-side off a SUBSTR of summary or description, so the big columns never cross into the response. The reader is unaffected: it already fetched the whole role by uid when it opened one; the reader pane's own reliance on the list row carrying the body is removed too, so it no longer flashes a wrong "no description" before that fetch lands. Measured on a copy of the live database: a 60-row page went from 392 KiB to 39 KiB, and a 1000-row restore from 6.6 MB to 631 KiB. Co-Authored-By: Claude Sonnet --- app/models.py | 59 ++++++++++++++++++++- app/static/roles.js | 40 +++++++++----- app/storage/store.py | 65 ++++++++++++++++++++++- app/web/api.py | 13 +++-- docs/decisions.md | 2 + tests/test_web.py | 121 +++++++++++++++++++++++++++++++++---------- 6 files changed, 251 insertions(+), 49 deletions(-) diff --git a/app/models.py b/app/models.py index 46606d0..941ea85 100644 --- a/app/models.py +++ b/app/models.py @@ -388,13 +388,68 @@ class ExtractResult(ModelTaskResult): class ScoredRole(Role): - """A role as `/api/roles` serves it: with one agent's score and gate - reason when an agent was asked for, and without otherwise.""" + """A role as `/api/roles/{uid}` serves it: with one agent's score and + gate reason when an agent was asked for, and without otherwise.""" score: float | None = None gated_reason: str = "" +#: How many characters `RoleRow.excerpt` keeps at most: the feed row's own +#: cap, built once here rather than by the client, so a list response never +#: carries more of a role's text than a row draws. +EXCERPT_LIMIT = 200 + + +def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str: + """`text` (a role's summary, or the opening of its description) with its + whitespace collapsed, cut to `limit` characters at a word boundary with + an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a + `SUBSTR` of the source column rather than the whole of it, so the feed's + list never carries what only the reader needs.""" + collapsed = " ".join((text or "").split()) + if len(collapsed) <= limit: + return collapsed + cut = collapsed[:limit] + space = cut.rfind(" ") + trimmed = cut[:space] if space > limit * 0.6 else cut + return trimmed.rstrip(" ,.;:" + chr(0x2013) + "-") + "…" + + +class RoleRow(Model): + """One row of the feed: what `GET /api/roles` serves per role, the + fields a row draws and nothing else — never `description`, `feed_text`, + `raw` or `article_text`, the four columns docs/decisions.md measures at + about 5.5 KB a role between them. The reader fetches the rest by uid when + it opens a role (`GET /api/roles/{uid}`, the full `Role`). + + `excerpt` is built on the server (see `excerpt`, above): the summary when + there is one, else the description's opening, cut to `EXCERPT_LIMIT` + characters. `salary_min`, `salary_max` and `currency` are the pay fields + a row already draws; `salary_text` and `salary_state`, once the role + model carries them, belong here too.""" + + uid: str + company_key: str = "" + company: str = "" + title: str = "" + place: str = "" + country: str = "" + remote: Remote = "" + function: str = "" + salary_min: int | None = None + salary_max: int | None = None + currency: str = "" + posted_at: str = "" + score: float | None = None + gated_reason: str = "" + vote: Literal[-1, 0, 1] | None = None + voted_at: str | None = None + read_at: str | None = None + tags: list[str] = [] + excerpt: str = "" + + class Company(Model): """A row of `companies`, with `affinity` as the agent it was read for has it (see `agent_companies`); 0 when no agent was asked for.""" diff --git a/app/static/roles.js b/app/static/roles.js index 8180ea9..ca76288 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -473,15 +473,23 @@ for (const el of had.values()) el.remove(); } - /* The summary when there is one, else the description's opening, cut at a - word. */ + /* Built on the server (`app.models.excerpt`) and served as `role.excerpt`, + so the list response never carries the full description a row would + otherwise need to cut this from — see docs/decisions.md, "The pages". */ function excerpt(role) { - if (role.summary) return role.summary; - const text = (role.description || "").replace(/\s+/g, " ").trim(); - if (text.length <= 160) return text; - const cut = text.slice(0, 160); + return role.excerpt || ""; + } + + /* The same cut, run here instead: for the one row `merge` refreshes after + a fetch or a summarise, off the full role that call answers with, which + is the only time this page ever sees a role's `summary` or + `description` directly. */ + function excerptOf(role) { + const text = (role.summary || role.description || "").replace(/\s+/g, " ").trim(); + if (text.length <= 200) return text; + const cut = text.slice(0, 200); const space = cut.lastIndexOf(" "); - return `${(space > 100 ? cut.slice(0, space) : cut).replace(/[\s,.;:–-]+$/, "")}…`; + return `${(space > 120 ? cut.slice(0, space) : cut).replace(/[\s,.;:–-]+$/, "")}…`; } function tagChips(tags) { @@ -723,9 +731,13 @@ } if (!open) { el.replaceChildren(); return; } - // The list row is already in memory, and a list row is the whole role, so - // the pane paints from it at once and the fetch only brings it up to date. + // The list row is already in memory, and carries what a row draws — + // title, company, place, tags, pay — so the pane paints those at once; + // the body, the summary state and the original link wait on the fetch + // (`loaded`), since the list row no longer carries them (see + // `Store.roles_page`). const stub = state.items.find((r) => r.uid === state.readingUid); + const loaded = Boolean(state.reading) && state.reading.uid === state.readingUid; const r = state.reading || stub || {}; const same = el.dataset.uid === state.readingUid; const scrolled = same ? el.querySelector(".reader-body")?.scrollTop || 0 : 0; @@ -736,13 +748,13 @@ const text = (r.article_text || r.description || "").trim(); const body = error ? `

This role could not be loaded: ${esc(error)}

` - : !r.uid + : !loaded ? `

Loading…

` : text ? paragraphs(text) : `

The board gave no description for this one.

`; const pay = salary(r); const waiting = state.summarising === state.readingUid; - const teaser = r.uid && !r.article_text && out && (r.description || "").length < TEASER; + const teaser = loaded && !r.article_text && out && (r.description || "").length < TEASER; el.innerHTML = `
@@ -767,7 +779,7 @@ ${r.summary ? `

Summary

${esc(r.summary)}

` : ""}
- ${r.uid && !r.summary && !waiting + ${loaded && !r.summary && !waiting ? `` : ""} ${teaser ? `` : ""} ${waiting @@ -819,9 +831,9 @@ else state.reading = updated; const item = state.items.find((x) => x.uid === updated.uid); if (item) { - Object.assign(item, { summary: updated.summary, article_text: updated.article_text }); + item.excerpt = excerptOf(updated); const row = $(`#roles-list .row[data-uid="${CSS.escape(updated.uid)}"]`); - if (row) row.role = null; // drawn again from the new text + if (row) row.role = null; // drawn again from the new excerpt renderFeed(); } renderReader(); diff --git a/app/storage/store.py b/app/storage/store.py index a37b3b1..4f7b477 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -22,8 +22,8 @@ import numpy as np from desk.models import Item from app.models import ( AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, - Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key, - country_of, place_key, posted_at, posted_host, remote_of, title_key, + Company, Gate, Limits, Role, RoleRow, ScoredRole, Subscription, Weights, company_key, + country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key, ) from app.storage import schema @@ -70,6 +70,17 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 " #: nothing, so `vote` and `voted_at` are NULL: one query shape either way. _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?" +#: What `roles_page` reads for `RoleRow.excerpt`: the summary when there is +#: one, else the description's opening. A `SUBSTR` of the source column, not +#: the column itself, so a big `description` never leaves SQLite whole for a +#: row that only ever shows `EXCERPT_LIMIT` characters of it. +_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)" + +#: How many raw characters `roles_page` pulls before `app.models.excerpt` +#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim +#: still has somewhere to land. +_EXCERPT_RAW = 240 + _COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city", "ats_slug") @@ -521,6 +532,56 @@ class Store: r.tags = by_uid.get(r.uid, []) return found + def roles_page(self, limit: int = 100, company: str | None = None, + place: str | None = None, open_only: bool = True, + q: str | None = None, function: str | None = None, + remote: str | None = None, days: int | None = None, + hide_muted: bool = False, include_read: bool = True, + tags: Sequence[str] = (), tag_mode: str = "any", + agent: int | None = None, by_score: bool = True, gated: bool = False, + judged: str | None = None, offset: int = 0) -> list[RoleRow]: + """What `GET /api/roles` serves: the same rows `roles` would, in the + same order, with the same filters and paging, but only the fields a + row draws (see `RoleRow`) — never `description`, `feed_text`, `raw` + or `article_text`, which a full `roles.*` select would carry whether + or not a page ever renders them. The reader still gets the whole role + by uid, see `role`. + + `excerpt` is `app.models.excerpt` over a `SUBSTR` of `_EXCERPT_SOURCE` + (`_EXCERPT_RAW` characters, trimmed further in Python), so the source + column itself never crosses out of SQLite for a list.""" + sql, params = self._roles_from( + company=company, place=place, open_only=open_only, q=q, function=function, + remote=remote, days=days, hide_muted=hide_muted, include_read=include_read, + tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated, + judged=judged) + select = ( + "SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, " + "roles.country, roles.remote, roles.function, roles.salary_min, " + "roles.salary_max, roles.currency, roles.posted_at, roles.read_at, " + "v.vote, v.voted_at, " + f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt") + order = "roles.posted_at DESC, roles.uid" + scored = agent is not None and by_score + if scored: + select += f", {_VOTED_SCORE} AS score, s.gated_reason" + order = "score DESC, " + order + elif open_only: + # Same reasoning as `roles`: the planner otherwise sorts every + # open role before applying LIMIT. + sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1) + rows = self._rows(f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", + [*params, limit, offset]) + found = [] + for r in rows: + row = dict(r) + row["excerpt"] = excerpt(row["excerpt"]) + found.append(RoleRow.from_row(row)) + by_uid = self._tags_by_uid([r.uid for r in found]) + for r in found: + r.tags = by_uid.get(r.uid, []) + return found + def count_roles(self, company: str | None = None, place: str | None = None, open_only: bool = True, q: str | None = None, function: str | None = None, remote: str | None = None, diff --git a/app/web/api.py b/app/web/api.py index 633dc59..8e365ac 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -16,7 +16,7 @@ from pydantic import BaseModel, Field from app import jobs from app.models import ( - AFFINITY, Agent, Application, Company, Gate, Limits, Role, ScoredRole, Status, + AFFINITY, Agent, Application, Company, Gate, Limits, Role, RoleRow, Status, Subscription, Weights, company_key, ) from app.storage import Store @@ -111,9 +111,14 @@ def unread_role(uid: str, store: StoreDep) -> Role: class RolesPage(BaseModel): - """One page of roles, and how many the same filters match in all.""" + """One page of roles, and how many the same filters match in all. `items` + is `RoleRow`, not the full `Role`: the fields a feed row draws, with an + excerpt built on the server, never the big columns (`description`, + `feed_text`, `raw`, `article_text`) a full role carries — see + `Store.roles_page`. The reader fetches the rest by uid when it opens one, + `GET /api/roles/{uid}`.""" - items: list[ScoredRole] + items: list[RoleRow] total: int @@ -161,7 +166,7 @@ def list_roles( "hide_muted": not show_muted, "include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent_id, "by_score": by_score, "gated": gated, "judged": judged or None} - return RolesPage(items=store.roles(limit=limit, offset=offset, **filters), + return RolesPage(items=store.roles_page(limit=limit, offset=offset, **filters), total=store.count_roles(**filters)) diff --git a/docs/decisions.md b/docs/decisions.md index 02218be..d7c8a5e 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -126,6 +126,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **A muted company is excluded in SQL, before `LIMIT`.** Filtering after `LIMIT` shipped first and showed nothing once Databricks' 884 freshly pulled roles filled the newest 200 rows and Databricks was muted. +**`GET /api/roles` serves a row, not a role.** It used to answer `roles.*`, the whole `Role` — `description`, `feed_text`, `raw` and `article_text` included, about 5.5 KB a role on average — for a feed that only ever shows a title, a few badges and an excerpt; a 1000-row restore was several megabytes of text a phone never renders. `Store.roles_page` selects only the fields a row draws (`RoleRow`), with `excerpt` built by `app.models.excerpt` off a `SUBSTR` of `summary` or `description`, never the column whole. Measured on a copy of the live database: a 60-row page went from 392 KiB to 39 KiB and a 1000-row restore from 6.6 MB to 631 KiB, both about a tenth. The reader is unaffected: it already fetched the whole role by uid when it opened one (`GET /api/roles/{uid}`, still the full `Role`); what changed is that the list row it painted from first, before that fetch lands, no longer carries a body to paint with, so the reader now waits on the fetch for the posting text and the original link the same way it always waited for the summary. + ## The phone **Two tabs and a cog.** `TABS` in `app/web/pages.py` becomes Roles and Applications; the other six pages (Agents, Companies, Sources, Jobs, Schedules, Messages, Account) move behind a settings cog at `/settings/`, because nine tabs wrapped to three rows on a phone and six of them were setup or system pages. diff --git a/tests/test_web.py b/tests/test_web.py index 2344f68..223e536 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -16,6 +16,7 @@ from fastapi.testclient import TestClient from app.config import Config, ServerConfig from app.main import create_app +from app.models import excerpt as models_excerpt from app.storage import Store from app.web import SETTINGS, TABS from app.web.pages import SECTIONS @@ -76,16 +77,17 @@ def test_list_roles_filters_by_query_place_and_function( store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering")) store.upsert_role(posting("2", company="Acme", place="Munich", function="sales")) store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering")) + uid = {r.source_id: r.uid for r in store.roles()} res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"}) assert res.status_code == 200 - assert {r["source_id"] for r in res.json()["items"]} == {"1", "3"} + assert {r["uid"] for r in res.json()["items"]} == {uid["1"], uid["3"]} res = client.get("/api/roles", params={"function": "sales"}) - assert [r["source_id"] for r in res.json()["items"]] == ["2"] + assert [r["uid"] for r in res.json()["items"]] == [uid["2"]] res = client.get("/api/roles", params={"company": "other"}) - assert [r["source_id"] for r in res.json()["items"]] == ["3"] + assert [r["uid"] for r in res.json()["items"]] == [uid["3"]] def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None: @@ -102,20 +104,21 @@ def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Sto store.upsert_role(posting("2", company="Other")) agent = store.save_agent("A") other = store.save_agent("B") + uid = {r.source_id: r.uid for r in store.roles()} def shown(**params: Any) -> set[str]: - return {r["source_id"] for r in client.get("/api/roles", params=params).json()["items"]} + return {r["uid"] for r in client.get("/api/roles", params=params).json()["items"]} - assert shown() == {"1", "2"} + assert shown() == {uid["1"], uid["2"]} # No agent in the body: the default agent's, which is also who a list # without `agent` hides for. assert client.post("/api/companies/acme/affinity", json={"affinity": -1}).status_code == 200 - assert shown() == {"2"} - assert shown(show_muted="true") == {"1", "2"} - assert shown(agent=agent.id, order="newest") == {"2"} + assert shown() == {uid["2"]} + assert shown(show_muted="true") == {uid["1"], uid["2"]} + assert shown(agent=agent.id, order="newest") == {uid["2"]} # Hidden is per agent: the other agent still sees Acme. - assert shown(agent=other.id, order="newest") == {"1", "2"} + assert shown(agent=other.id, order="newest") == {uid["1"], uid["2"]} assert [c["affinity"] for c in client.get("/api/companies").json()] == [-1] assert [c["affinity"] for c in client.get("/api/companies", params={"agent": other.id}).json()] == [0] @@ -136,9 +139,10 @@ def test_muted_is_excluded_before_the_limit_not_after( for i in range(3): store.upsert_role(posting(f"a{i}", company="Acme")) clock.tick() + uid = {r.source_id: r.uid for r in store.roles()} res = client.get("/api/roles", params={"limit": 1}) - assert [r["source_id"] for r in res.json()["items"]] == ["other"] + assert [r["uid"] for r in res.json()["items"]] == [uid["other"]] assert res.json()["total"] == 1 @@ -149,12 +153,13 @@ def test_list_roles_pages_by_offset_and_counts_the_total( for i in range(5): store.upsert_role(posting(str(i))) clock.tick() + uid = {r.source_id: r.uid for r in store.roles()} first = client.get("/api/roles", params={"limit": 2}).json() - assert [r["source_id"] for r in first["items"]] == ["4", "3"] + assert [r["uid"] for r in first["items"]] == [uid["4"], uid["3"]] assert first["total"] == 5 rest = client.get("/api/roles", params={"limit": 2, "offset": 4}).json() - assert [r["source_id"] for r in rest["items"]] == ["0"] and rest["total"] == 5 + assert [r["uid"] for r in rest["items"]] == [uid["0"]] and rest["total"] == 5 assert client.get("/api/roles", params={"offset": 9}).json() == {"items": [], "total": 5} assert client.get("/api/roles", params={"offset": -1}).status_code == 422 assert len(client.get("/api/roles").json()["items"]) == 5 # default 60 @@ -176,13 +181,13 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) def judged(value: str, **params: Any) -> set[str]: page = client.get("/api/roles", params={"judged": value, **params}).json() - return {r["source_id"] for r in page["items"]} + return {r["uid"] for r in page["items"]} - assert judged("liked") == {"liked"} - assert judged("passed") == {"passed"} - assert judged("unjudged") == {"open"} - assert judged("") == {"liked", "passed", "fine", "open"} # a pass is not hidden - assert judged("liked", agent=agent.id, order="newest") == {"liked"} + assert judged("liked") == {uid["liked"]} + assert judged("passed") == {uid["passed"]} + assert judged("unjudged") == {uid["open"]} + assert judged("") == {uid["liked"], uid["passed"], uid["fine"], uid["open"]} # a pass is not hidden + assert judged("liked", agent=agent.id, order="newest") == {uid["liked"]} assert client.get("/api/roles", params={"judged": "maybe"}).status_code == 422 assert client.get(f"/api/roles/{uid['liked']}").json()["vote"] == 1 assert client.get(f"/api/roles/{uid['liked']}", params={"agent": agent.id}).json()["vote"] == 1 @@ -190,7 +195,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) cleared = client.delete(f"/api/roles/{uid['liked']}/vote") assert cleared.status_code == 200 assert cleared.json()["vote"] is None and cleared.json()["voted_at"] is None - assert judged("unjudged") == {"open", "liked"} + assert judged("unjudged") == {uid["open"], uid["liked"]} cleared = client.delete(f"/api/roles/{uid['passed']}/vote", params={"agent": agent.id}) assert cleared.status_code == 200 and cleared.json()["vote"] is None @@ -254,12 +259,12 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes( store.set_vote(agent.id, uid["1"], 1) page = client.get("/api/roles", params={"agent": agent.id, "order": "newest"}).json() - assert [r["source_id"] for r in page["items"]] == ["3", "2", "1"] + assert [r["uid"] for r in page["items"]] == [uid["3"], uid["2"], uid["1"]] assert [r["score"] for r in page["items"]] == [None, None, None] assert [r["vote"] for r in page["items"]] == [None, None, 1] assert page["total"] == 3 scored = client.get("/api/roles", params={"agent": agent.id, "order": "score"}).json() - assert [r["source_id"] for r in scored["items"]] == ["1", "2"] + assert [r["uid"] for r in scored["items"]] == [uid["1"], uid["2"]] assert client.get("/api/roles", params={"agent": agent.id, "order": "best"}).status_code == 422 @@ -269,13 +274,14 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once( for sid in ("a", "b", "c"): store.upsert_role(posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} + label = {v: k for k, v in uid.items()} agent = store.save_agent("A") store.replace_scores(agent.id, [(uid["a"], 0.9, ""), (uid["b"], 0.6, ""), (uid["c"], 0.55, "")]) def ranked() -> list[tuple[str, float]]: page = client.get("/api/roles", params={"agent": agent.id}).json() - return [(r["source_id"], round(r["score"], 3)) for r in page["items"]] + return [(label[r["uid"]], round(r["score"], 3)) for r in page["items"]] assert ranked() == [("a", 0.9), ("b", 0.6), ("c", 0.55)] client.post(f"/api/roles/{uid['a']}/vote", json={"value": -1}) @@ -305,14 +311,15 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None: store.upsert_role(posting("1")) store.upsert_role(posting("2")) + ids = {r.source_id: r.uid for r in store.roles()} uid = posting("1").uid client.post(f"/api/roles/{uid}/read") all_ = client.get("/api/roles").json() - assert {r["source_id"] for r in all_["items"]} == {"1", "2"} + assert {r["uid"] for r in all_["items"]} == {ids["1"], ids["2"]} unread = client.get("/api/roles", params={"include_read": "false"}).json() - assert {r["source_id"] for r in unread["items"]} == {"2"} + assert {r["uid"] for r in unread["items"]} == {ids["2"]} assert unread["total"] == 1 @@ -324,11 +331,11 @@ def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store store.set_tags(uid["2"], ["python", "frontend"]) res = client.get("/api/roles", params={"tags": ["python"]}) - assert {r["source_id"] for r in res.json()["items"]} == {"1", "2"} + assert {r["uid"] for r in res.json()["items"]} == {uid["1"], uid["2"]} both = client.get("/api/roles", params={"tags": ["python", "backend"], "tag_mode": "all"}) - assert {r["source_id"] for r in both.json()["items"]} == {"1"} + assert {r["uid"] for r in both.json()["items"]} == {uid["1"]} [item] = both.json()["items"] assert item["tags"] == ["backend", "python"] @@ -831,7 +838,7 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated( (uids["gated"], 0.0, "place unknown")]) shown = client.get(f"/api/roles?agent={agent.id}").json()["items"] - assert [r["source_id"] for r in shown] == ["high", "low"] + assert [r["uid"] for r in shown] == [uids["high"], uids["low"]] assert shown[0]["score"] == 0.8 everything = client.get(f"/api/roles?agent={agent.id}&gated=1").json()["items"] assert everything[-1]["gated_reason"] == "place unknown" @@ -839,6 +846,66 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated( assert client.get("/api/roles?agent=999").status_code == 404 +def test_list_roles_serves_the_row_projection_and_not_the_big_columns( + client: TestClient, store: Store) -> None: + """`GET /api/roles` is `RoleRow`: the fields a feed row draws, an + `excerpt` built on the server, and never `description`, `feed_text`, + `raw` or `article_text` — those are what the reader's own fetch, + `GET /api/roles/{uid}`, still answers in full.""" + long_description = "Build things. " * 50 + item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid", + function="engineering") + item.body = long_description + store.upsert_role(item) + uid = item.uid + + row = client.get("/api/roles").json()["items"][0] + assert set(row) == { + "uid", "company_key", "company", "title", "place", "country", "remote", "function", + "salary_min", "salary_max", "currency", "posted_at", "score", "gated_reason", "vote", + "voted_at", "read_at", "tags", "excerpt", + } + assert row["uid"] == uid + assert row["company"] == "Acme" and row["place"] == "Berlin, Germany" + assert row["remote"] == "hybrid" and row["function"] == "engineering" + assert len(row["excerpt"]) <= 200 + assert row["excerpt"] == models_excerpt(long_description) + + full = client.get(f"/api/roles/{uid}").json() + assert full["description"] == long_description + + +def test_list_roles_page_stays_small_even_with_long_postings( + client: TestClient, store: Store) -> None: + """The size bound the projection exists for: a page of rows with a real + posting's worth of text each (description, feed_text, raw and + article_text set, as a board's upsert would leave them) stays small, + because none of the four ever reach the response. Measured against a + copy of the live database in the report; this is the regression guard.""" + big = "Requirement. " * 500 # ~6.5 KB, decisions.md's own average + for n in range(60): + item = posting(str(n), company="Acme") + item.body = big + store.upsert_role(item) + with store.transaction() as conn: + conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?", + (big, json.dumps({"blob": big}), big)) + + page = client.get("/api/roles", params={"limit": 60}) + assert len(page.json()["items"]) == 60 + # Well under one tenth of what 60 full roles (about 5.5 KB each, + # decisions.md's own figure) would answer with. + assert len(page.content) < 100_000 + + +def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None: + store.upsert_role(posting("1")) + uid = posting("1").uid + store.set_summary(uid, "A short, hand-written summary.") + row = client.get("/api/roles").json()["items"][0] + assert row["excerpt"] == "A short, hand-written summary." + + # ----------------------------------------------------------------- sources def _fake_search(monkeypatch: pytest.MonkeyPatch, rows: int) -> None: