Serve a row projection from GET /api/roles instead of the whole role

The feed only ever shows a title, a few badges and a short excerpt,
but the list endpoint answered with every role's full row --
description, feed_text, raw and article_text included, about 5.5 KB
a role -- so a 1000-row restore was several megabytes of text a phone
never renders.

Store.roles_page selects only the fields a row draws (RoleRow), with
excerpt built server-side off a SUBSTR of summary or description, so
the big columns never cross into the response. The reader is
unaffected: it already fetched the whole role by uid when it opened
one; the reader pane's own reliance on the list row carrying the body
is removed too, so it no longer flashes a wrong "no description"
before that fetch lands.

Measured on a copy of the live database: a 60-row page went from
392 KiB to 39 KiB, and a 1000-row restore from 6.6 MB to 631 KiB.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet committed 2026-09-29 15:22:00 +02:00
1 parent e6be593542
commit 91596300e1
6 files changed
+251 -49

No files matched your search

+94 -27
View File
@@ -16,6 +16,7 @@ from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
from app.models import excerpt as models_excerpt
from app.storage import Store
from app.web import SETTINGS, TABS
from app.web.pages import SECTIONS
@@ -76,16 +77,17 @@ def test_list_roles_filters_by_query_place_and_function(
store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering"))
store.upsert_role(posting("2", company="Acme", place="Munich", function="sales"))
store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering"))
uid = {r.source_id: r.uid for r in store.roles()}
res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"})
assert res.status_code == 200
assert {r["source_id"] for r in res.json()["items"]} == {"1", "3"}
assert {r["uid"] for r in res.json()["items"]} == {uid["1"], uid["3"]}
res = client.get("/api/roles", params={"function": "sales"})
assert [r["source_id"] for r in res.json()["items"]] == ["2"]
assert [r["uid"] for r in res.json()["items"]] == [uid["2"]]
res = client.get("/api/roles", params={"company": "other"})
assert [r["source_id"] for r in res.json()["items"]] == ["3"]
assert [r["uid"] for r in res.json()["items"]] == [uid["3"]]
def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
@@ -102,20 +104,21 @@ def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Sto
store.upsert_role(posting("2", company="Other"))
agent = store.save_agent("A")
other = store.save_agent("B")
uid = {r.source_id: r.uid for r in store.roles()}
def shown(**params: Any) -> set[str]:
return {r["source_id"] for r in client.get("/api/roles", params=params).json()["items"]}
return {r["uid"] for r in client.get("/api/roles", params=params).json()["items"]}
assert shown() == {"1", "2"}
assert shown() == {uid["1"], uid["2"]}
# No agent in the body: the default agent's, which is also who a list
# without `agent` hides for.
assert client.post("/api/companies/acme/affinity", json={"affinity": -1}).status_code == 200
assert shown() == {"2"}
assert shown(show_muted="true") == {"1", "2"}
assert shown(agent=agent.id, order="newest") == {"2"}
assert shown() == {uid["2"]}
assert shown(show_muted="true") == {uid["1"], uid["2"]}
assert shown(agent=agent.id, order="newest") == {uid["2"]}
# Hidden is per agent: the other agent still sees Acme.
assert shown(agent=other.id, order="newest") == {"1", "2"}
assert shown(agent=other.id, order="newest") == {uid["1"], uid["2"]}
assert [c["affinity"] for c in client.get("/api/companies").json()] == [-1]
assert [c["affinity"] for c in
client.get("/api/companies", params={"agent": other.id}).json()] == [0]
@@ -136,9 +139,10 @@ def test_muted_is_excluded_before_the_limit_not_after(
for i in range(3):
store.upsert_role(posting(f"a{i}", company="Acme"))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
res = client.get("/api/roles", params={"limit": 1})
assert [r["source_id"] for r in res.json()["items"]] == ["other"]
assert [r["uid"] for r in res.json()["items"]] == [uid["other"]]
assert res.json()["total"] == 1
@@ -149,12 +153,13 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
for i in range(5):
store.upsert_role(posting(str(i)))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
first = client.get("/api/roles", params={"limit": 2}).json()
assert [r["source_id"] for r in first["items"]] == ["4", "3"]
assert [r["uid"] for r in first["items"]] == [uid["4"], uid["3"]]
assert first["total"] == 5
rest = client.get("/api/roles", params={"limit": 2, "offset": 4}).json()
assert [r["source_id"] for r in rest["items"]] == ["0"] and rest["total"] == 5
assert [r["uid"] for r in rest["items"]] == [uid["0"]] and rest["total"] == 5
assert client.get("/api/roles", params={"offset": 9}).json() == {"items": [], "total": 5}
assert client.get("/api/roles", params={"offset": -1}).status_code == 422
assert len(client.get("/api/roles").json()["items"]) == 5 # default 60
@@ -176,13 +181,13 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
def judged(value: str, **params: Any) -> set[str]:
page = client.get("/api/roles", params={"judged": value, **params}).json()
return {r["source_id"] for r in page["items"]}
return {r["uid"] for r in page["items"]}
assert judged("liked") == {"liked"}
assert judged("passed") == {"passed"}
assert judged("unjudged") == {"open"}
assert judged("") == {"liked", "passed", "fine", "open"} # a pass is not hidden
assert judged("liked", agent=agent.id, order="newest") == {"liked"}
assert judged("liked") == {uid["liked"]}
assert judged("passed") == {uid["passed"]}
assert judged("unjudged") == {uid["open"]}
assert judged("") == {uid["liked"], uid["passed"], uid["fine"], uid["open"]} # a pass is not hidden
assert judged("liked", agent=agent.id, order="newest") == {uid["liked"]}
assert client.get("/api/roles", params={"judged": "maybe"}).status_code == 422
assert client.get(f"/api/roles/{uid['liked']}").json()["vote"] == 1
assert client.get(f"/api/roles/{uid['liked']}", params={"agent": agent.id}).json()["vote"] == 1
@@ -190,7 +195,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
cleared = client.delete(f"/api/roles/{uid['liked']}/vote")
assert cleared.status_code == 200
assert cleared.json()["vote"] is None and cleared.json()["voted_at"] is None
assert judged("unjudged") == {"open", "liked"}
assert judged("unjudged") == {uid["open"], uid["liked"]}
cleared = client.delete(f"/api/roles/{uid['passed']}/vote", params={"agent": agent.id})
assert cleared.status_code == 200 and cleared.json()["vote"] is None
@@ -254,12 +259,12 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
store.set_vote(agent.id, uid["1"], 1)
page = client.get("/api/roles", params={"agent": agent.id, "order": "newest"}).json()
assert [r["source_id"] for r in page["items"]] == ["3", "2", "1"]
assert [r["uid"] for r in page["items"]] == [uid["3"], uid["2"], uid["1"]]
assert [r["score"] for r in page["items"]] == [None, None, None]
assert [r["vote"] for r in page["items"]] == [None, None, 1]
assert page["total"] == 3
scored = client.get("/api/roles", params={"agent": agent.id, "order": "score"}).json()
assert [r["source_id"] for r in scored["items"]] == ["1", "2"]
assert [r["uid"] for r in scored["items"]] == [uid["1"], uid["2"]]
assert client.get("/api/roles", params={"agent": agent.id,
"order": "best"}).status_code == 422
@@ -269,13 +274,14 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once(
for sid in ("a", "b", "c"):
store.upsert_role(posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
label = {v: k for k, v in uid.items()}
agent = store.save_agent("A")
store.replace_scores(agent.id, [(uid["a"], 0.9, ""), (uid["b"], 0.6, ""),
(uid["c"], 0.55, "")])
def ranked() -> list[tuple[str, float]]:
page = client.get("/api/roles", params={"agent": agent.id}).json()
return [(r["source_id"], round(r["score"], 3)) for r in page["items"]]
return [(label[r["uid"]], round(r["score"], 3)) for r in page["items"]]
assert ranked() == [("a", 0.9), ("b", 0.6), ("c", 0.55)]
client.post(f"/api/roles/{uid['a']}/vote", json={"value": -1})
@@ -305,14 +311,15 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S
def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
ids = {r.source_id: r.uid for r in store.roles()}
uid = posting("1").uid
client.post(f"/api/roles/{uid}/read")
all_ = client.get("/api/roles").json()
assert {r["source_id"] for r in all_["items"]} == {"1", "2"}
assert {r["uid"] for r in all_["items"]} == {ids["1"], ids["2"]}
unread = client.get("/api/roles", params={"include_read": "false"}).json()
assert {r["source_id"] for r in unread["items"]} == {"2"}
assert {r["uid"] for r in unread["items"]} == {ids["2"]}
assert unread["total"] == 1
@@ -324,11 +331,11 @@ def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store
store.set_tags(uid["2"], ["python", "frontend"])
res = client.get("/api/roles", params={"tags": ["python"]})
assert {r["source_id"] for r in res.json()["items"]} == {"1", "2"}
assert {r["uid"] for r in res.json()["items"]} == {uid["1"], uid["2"]}
both = client.get("/api/roles", params={"tags": ["python", "backend"],
"tag_mode": "all"})
assert {r["source_id"] for r in both.json()["items"]} == {"1"}
assert {r["uid"] for r in both.json()["items"]} == {uid["1"]}
[item] = both.json()["items"]
assert item["tags"] == ["backend", "python"]
@@ -831,7 +838,7 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
(uids["gated"], 0.0, "place unknown")])
shown = client.get(f"/api/roles?agent={agent.id}").json()["items"]
assert [r["source_id"] for r in shown] == ["high", "low"]
assert [r["uid"] for r in shown] == [uids["high"], uids["low"]]
assert shown[0]["score"] == 0.8
everything = client.get(f"/api/roles?agent={agent.id}&gated=1").json()["items"]
assert everything[-1]["gated_reason"] == "place unknown"
@@ -839,6 +846,66 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
assert client.get("/api/roles?agent=999").status_code == 404
def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
client: TestClient, store: Store) -> None:
"""`GET /api/roles` is `RoleRow`: the fields a feed row draws, an
`excerpt` built on the server, and never `description`, `feed_text`,
`raw` or `article_text` — those are what the reader's own fetch,
`GET /api/roles/{uid}`, still answers in full."""
long_description = "Build things. " * 50
item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid",
function="engineering")
item.body = long_description
store.upsert_role(item)
uid = item.uid
row = client.get("/api/roles").json()["items"][0]
assert set(row) == {
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
"salary_min", "salary_max", "currency", "posted_at", "score", "gated_reason", "vote",
"voted_at", "read_at", "tags", "excerpt",
}
assert row["uid"] == uid
assert row["company"] == "Acme" and row["place"] == "Berlin, Germany"
assert row["remote"] == "hybrid" and row["function"] == "engineering"
assert len(row["excerpt"]) <= 200
assert row["excerpt"] == models_excerpt(long_description)
full = client.get(f"/api/roles/{uid}").json()
assert full["description"] == long_description
def test_list_roles_page_stays_small_even_with_long_postings(
client: TestClient, store: Store) -> None:
"""The size bound the projection exists for: a page of rows with a real
posting's worth of text each (description, feed_text, raw and
article_text set, as a board's upsert would leave them) stays small,
because none of the four ever reach the response. Measured against a
copy of the live database in the report; this is the regression guard."""
big = "Requirement. " * 500 # ~6.5 KB, decisions.md's own average
for n in range(60):
item = posting(str(n), company="Acme")
item.body = big
store.upsert_role(item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?",
(big, json.dumps({"blob": big}), big))
page = client.get("/api/roles", params={"limit": 60})
assert len(page.json()["items"]) == 60
# Well under one tenth of what 60 full roles (about 5.5 KB each,
# decisions.md's own figure) would answer with.
assert len(page.content) < 100_000
def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
uid = posting("1").uid
store.set_summary(uid, "A short, hand-written summary.")
row = client.get("/api/roles").json()["items"][0]
assert row["excerpt"] == "A short, hand-written summary."
# ----------------------------------------------------------------- sources
def _fake_search(monkeypatch: pytest.MonkeyPatch, rows: int) -> None: