Serve a row projection from GET /api/roles instead of the whole role
The feed only ever shows a title, a few badges and a short excerpt, but the list endpoint answered with every role's full row -- description, feed_text, raw and article_text included, about 5.5 KB a role -- so a 1000-row restore was several megabytes of text a phone never renders. Store.roles_page selects only the fields a row draws (RoleRow), with excerpt built server-side off a SUBSTR of summary or description, so the big columns never cross into the response. The reader is unaffected: it already fetched the whole role by uid when it opened one; the reader pane's own reliance on the list row carrying the body is removed too, so it no longer flashes a wrong "no description" before that fetch lands. Measured on a copy of the live database: a 60-row page went from 392 KiB to 39 KiB, and a 1000-row restore from 6.6 MB to 631 KiB. Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
1 parent
e6be593542
commit
91596300e1
6 files changed
+251
-49
No files matched your search
+94
-27
@@ -16,6 +16,7 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from app.config import Config, ServerConfig
|
||||
from app.main import create_app
|
||||
from app.models import excerpt as models_excerpt
|
||||
from app.storage import Store
|
||||
from app.web import SETTINGS, TABS
|
||||
from app.web.pages import SECTIONS
|
||||
@@ -76,16 +77,17 @@ def test_list_roles_filters_by_query_place_and_function(
|
||||
store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering"))
|
||||
store.upsert_role(posting("2", company="Acme", place="Munich", function="sales"))
|
||||
store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering"))
|
||||
uid = {r.source_id: r.uid for r in store.roles()}
|
||||
|
||||
res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"})
|
||||
assert res.status_code == 200
|
||||
assert {r["source_id"] for r in res.json()["items"]} == {"1", "3"}
|
||||
assert {r["uid"] for r in res.json()["items"]} == {uid["1"], uid["3"]}
|
||||
|
||||
res = client.get("/api/roles", params={"function": "sales"})
|
||||
assert [r["source_id"] for r in res.json()["items"]] == ["2"]
|
||||
assert [r["uid"] for r in res.json()["items"]] == [uid["2"]]
|
||||
|
||||
res = client.get("/api/roles", params={"company": "other"})
|
||||
assert [r["source_id"] for r in res.json()["items"]] == ["3"]
|
||||
assert [r["uid"] for r in res.json()["items"]] == [uid["3"]]
|
||||
|
||||
|
||||
def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
|
||||
@@ -102,20 +104,21 @@ def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Sto
|
||||
store.upsert_role(posting("2", company="Other"))
|
||||
agent = store.save_agent("A")
|
||||
other = store.save_agent("B")
|
||||
uid = {r.source_id: r.uid for r in store.roles()}
|
||||
|
||||
def shown(**params: Any) -> set[str]:
|
||||
return {r["source_id"] for r in client.get("/api/roles", params=params).json()["items"]}
|
||||
return {r["uid"] for r in client.get("/api/roles", params=params).json()["items"]}
|
||||
|
||||
assert shown() == {"1", "2"}
|
||||
assert shown() == {uid["1"], uid["2"]}
|
||||
|
||||
# No agent in the body: the default agent's, which is also who a list
|
||||
# without `agent` hides for.
|
||||
assert client.post("/api/companies/acme/affinity", json={"affinity": -1}).status_code == 200
|
||||
assert shown() == {"2"}
|
||||
assert shown(show_muted="true") == {"1", "2"}
|
||||
assert shown(agent=agent.id, order="newest") == {"2"}
|
||||
assert shown() == {uid["2"]}
|
||||
assert shown(show_muted="true") == {uid["1"], uid["2"]}
|
||||
assert shown(agent=agent.id, order="newest") == {uid["2"]}
|
||||
# Hidden is per agent: the other agent still sees Acme.
|
||||
assert shown(agent=other.id, order="newest") == {"1", "2"}
|
||||
assert shown(agent=other.id, order="newest") == {uid["1"], uid["2"]}
|
||||
assert [c["affinity"] for c in client.get("/api/companies").json()] == [-1]
|
||||
assert [c["affinity"] for c in
|
||||
client.get("/api/companies", params={"agent": other.id}).json()] == [0]
|
||||
@@ -136,9 +139,10 @@ def test_muted_is_excluded_before_the_limit_not_after(
|
||||
for i in range(3):
|
||||
store.upsert_role(posting(f"a{i}", company="Acme"))
|
||||
clock.tick()
|
||||
uid = {r.source_id: r.uid for r in store.roles()}
|
||||
|
||||
res = client.get("/api/roles", params={"limit": 1})
|
||||
assert [r["source_id"] for r in res.json()["items"]] == ["other"]
|
||||
assert [r["uid"] for r in res.json()["items"]] == [uid["other"]]
|
||||
assert res.json()["total"] == 1
|
||||
|
||||
|
||||
@@ -149,12 +153,13 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
|
||||
for i in range(5):
|
||||
store.upsert_role(posting(str(i)))
|
||||
clock.tick()
|
||||
uid = {r.source_id: r.uid for r in store.roles()}
|
||||
|
||||
first = client.get("/api/roles", params={"limit": 2}).json()
|
||||
assert [r["source_id"] for r in first["items"]] == ["4", "3"]
|
||||
assert [r["uid"] for r in first["items"]] == [uid["4"], uid["3"]]
|
||||
assert first["total"] == 5
|
||||
rest = client.get("/api/roles", params={"limit": 2, "offset": 4}).json()
|
||||
assert [r["source_id"] for r in rest["items"]] == ["0"] and rest["total"] == 5
|
||||
assert [r["uid"] for r in rest["items"]] == [uid["0"]] and rest["total"] == 5
|
||||
assert client.get("/api/roles", params={"offset": 9}).json() == {"items": [], "total": 5}
|
||||
assert client.get("/api/roles", params={"offset": -1}).status_code == 422
|
||||
assert len(client.get("/api/roles").json()["items"]) == 5 # default 60
|
||||
@@ -176,13 +181,13 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
|
||||
|
||||
def judged(value: str, **params: Any) -> set[str]:
|
||||
page = client.get("/api/roles", params={"judged": value, **params}).json()
|
||||
return {r["source_id"] for r in page["items"]}
|
||||
return {r["uid"] for r in page["items"]}
|
||||
|
||||
assert judged("liked") == {"liked"}
|
||||
assert judged("passed") == {"passed"}
|
||||
assert judged("unjudged") == {"open"}
|
||||
assert judged("") == {"liked", "passed", "fine", "open"} # a pass is not hidden
|
||||
assert judged("liked", agent=agent.id, order="newest") == {"liked"}
|
||||
assert judged("liked") == {uid["liked"]}
|
||||
assert judged("passed") == {uid["passed"]}
|
||||
assert judged("unjudged") == {uid["open"]}
|
||||
assert judged("") == {uid["liked"], uid["passed"], uid["fine"], uid["open"]} # a pass is not hidden
|
||||
assert judged("liked", agent=agent.id, order="newest") == {uid["liked"]}
|
||||
assert client.get("/api/roles", params={"judged": "maybe"}).status_code == 422
|
||||
assert client.get(f"/api/roles/{uid['liked']}").json()["vote"] == 1
|
||||
assert client.get(f"/api/roles/{uid['liked']}", params={"agent": agent.id}).json()["vote"] == 1
|
||||
@@ -190,7 +195,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
|
||||
cleared = client.delete(f"/api/roles/{uid['liked']}/vote")
|
||||
assert cleared.status_code == 200
|
||||
assert cleared.json()["vote"] is None and cleared.json()["voted_at"] is None
|
||||
assert judged("unjudged") == {"open", "liked"}
|
||||
assert judged("unjudged") == {uid["open"], uid["liked"]}
|
||||
cleared = client.delete(f"/api/roles/{uid['passed']}/vote", params={"agent": agent.id})
|
||||
assert cleared.status_code == 200 and cleared.json()["vote"] is None
|
||||
|
||||
@@ -254,12 +259,12 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
|
||||
store.set_vote(agent.id, uid["1"], 1)
|
||||
|
||||
page = client.get("/api/roles", params={"agent": agent.id, "order": "newest"}).json()
|
||||
assert [r["source_id"] for r in page["items"]] == ["3", "2", "1"]
|
||||
assert [r["uid"] for r in page["items"]] == [uid["3"], uid["2"], uid["1"]]
|
||||
assert [r["score"] for r in page["items"]] == [None, None, None]
|
||||
assert [r["vote"] for r in page["items"]] == [None, None, 1]
|
||||
assert page["total"] == 3
|
||||
scored = client.get("/api/roles", params={"agent": agent.id, "order": "score"}).json()
|
||||
assert [r["source_id"] for r in scored["items"]] == ["1", "2"]
|
||||
assert [r["uid"] for r in scored["items"]] == [uid["1"], uid["2"]]
|
||||
assert client.get("/api/roles", params={"agent": agent.id,
|
||||
"order": "best"}).status_code == 422
|
||||
|
||||
@@ -269,13 +274,14 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once(
|
||||
for sid in ("a", "b", "c"):
|
||||
store.upsert_role(posting(sid))
|
||||
uid = {r.source_id: r.uid for r in store.roles()}
|
||||
label = {v: k for k, v in uid.items()}
|
||||
agent = store.save_agent("A")
|
||||
store.replace_scores(agent.id, [(uid["a"], 0.9, ""), (uid["b"], 0.6, ""),
|
||||
(uid["c"], 0.55, "")])
|
||||
|
||||
def ranked() -> list[tuple[str, float]]:
|
||||
page = client.get("/api/roles", params={"agent": agent.id}).json()
|
||||
return [(r["source_id"], round(r["score"], 3)) for r in page["items"]]
|
||||
return [(label[r["uid"]], round(r["score"], 3)) for r in page["items"]]
|
||||
|
||||
assert ranked() == [("a", 0.9), ("b", 0.6), ("c", 0.55)]
|
||||
client.post(f"/api/roles/{uid['a']}/vote", json={"value": -1})
|
||||
@@ -305,14 +311,15 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S
|
||||
def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
store.upsert_role(posting("2"))
|
||||
ids = {r.source_id: r.uid for r in store.roles()}
|
||||
uid = posting("1").uid
|
||||
client.post(f"/api/roles/{uid}/read")
|
||||
|
||||
all_ = client.get("/api/roles").json()
|
||||
assert {r["source_id"] for r in all_["items"]} == {"1", "2"}
|
||||
assert {r["uid"] for r in all_["items"]} == {ids["1"], ids["2"]}
|
||||
|
||||
unread = client.get("/api/roles", params={"include_read": "false"}).json()
|
||||
assert {r["source_id"] for r in unread["items"]} == {"2"}
|
||||
assert {r["uid"] for r in unread["items"]} == {ids["2"]}
|
||||
assert unread["total"] == 1
|
||||
|
||||
|
||||
@@ -324,11 +331,11 @@ def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store
|
||||
store.set_tags(uid["2"], ["python", "frontend"])
|
||||
|
||||
res = client.get("/api/roles", params={"tags": ["python"]})
|
||||
assert {r["source_id"] for r in res.json()["items"]} == {"1", "2"}
|
||||
assert {r["uid"] for r in res.json()["items"]} == {uid["1"], uid["2"]}
|
||||
|
||||
both = client.get("/api/roles", params={"tags": ["python", "backend"],
|
||||
"tag_mode": "all"})
|
||||
assert {r["source_id"] for r in both.json()["items"]} == {"1"}
|
||||
assert {r["uid"] for r in both.json()["items"]} == {uid["1"]}
|
||||
[item] = both.json()["items"]
|
||||
assert item["tags"] == ["backend", "python"]
|
||||
|
||||
@@ -831,7 +838,7 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
|
||||
(uids["gated"], 0.0, "place unknown")])
|
||||
|
||||
shown = client.get(f"/api/roles?agent={agent.id}").json()["items"]
|
||||
assert [r["source_id"] for r in shown] == ["high", "low"]
|
||||
assert [r["uid"] for r in shown] == [uids["high"], uids["low"]]
|
||||
assert shown[0]["score"] == 0.8
|
||||
everything = client.get(f"/api/roles?agent={agent.id}&gated=1").json()["items"]
|
||||
assert everything[-1]["gated_reason"] == "place unknown"
|
||||
@@ -839,6 +846,66 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
|
||||
assert client.get("/api/roles?agent=999").status_code == 404
|
||||
|
||||
|
||||
def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
|
||||
client: TestClient, store: Store) -> None:
|
||||
"""`GET /api/roles` is `RoleRow`: the fields a feed row draws, an
|
||||
`excerpt` built on the server, and never `description`, `feed_text`,
|
||||
`raw` or `article_text` — those are what the reader's own fetch,
|
||||
`GET /api/roles/{uid}`, still answers in full."""
|
||||
long_description = "Build things. " * 50
|
||||
item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid",
|
||||
function="engineering")
|
||||
item.body = long_description
|
||||
store.upsert_role(item)
|
||||
uid = item.uid
|
||||
|
||||
row = client.get("/api/roles").json()["items"][0]
|
||||
assert set(row) == {
|
||||
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
|
||||
"salary_min", "salary_max", "currency", "posted_at", "score", "gated_reason", "vote",
|
||||
"voted_at", "read_at", "tags", "excerpt",
|
||||
}
|
||||
assert row["uid"] == uid
|
||||
assert row["company"] == "Acme" and row["place"] == "Berlin, Germany"
|
||||
assert row["remote"] == "hybrid" and row["function"] == "engineering"
|
||||
assert len(row["excerpt"]) <= 200
|
||||
assert row["excerpt"] == models_excerpt(long_description)
|
||||
|
||||
full = client.get(f"/api/roles/{uid}").json()
|
||||
assert full["description"] == long_description
|
||||
|
||||
|
||||
def test_list_roles_page_stays_small_even_with_long_postings(
|
||||
client: TestClient, store: Store) -> None:
|
||||
"""The size bound the projection exists for: a page of rows with a real
|
||||
posting's worth of text each (description, feed_text, raw and
|
||||
article_text set, as a board's upsert would leave them) stays small,
|
||||
because none of the four ever reach the response. Measured against a
|
||||
copy of the live database in the report; this is the regression guard."""
|
||||
big = "Requirement. " * 500 # ~6.5 KB, decisions.md's own average
|
||||
for n in range(60):
|
||||
item = posting(str(n), company="Acme")
|
||||
item.body = big
|
||||
store.upsert_role(item)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?",
|
||||
(big, json.dumps({"blob": big}), big))
|
||||
|
||||
page = client.get("/api/roles", params={"limit": 60})
|
||||
assert len(page.json()["items"]) == 60
|
||||
# Well under one tenth of what 60 full roles (about 5.5 KB each,
|
||||
# decisions.md's own figure) would answer with.
|
||||
assert len(page.content) < 100_000
|
||||
|
||||
|
||||
def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
uid = posting("1").uid
|
||||
store.set_summary(uid, "A short, hand-written summary.")
|
||||
row = client.get("/api/roles").json()["items"][0]
|
||||
assert row["excerpt"] == "A short, hand-written summary."
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- sources
|
||||
|
||||
def _fake_search(monkeypatch: pytest.MonkeyPatch, rows: int) -> None:
|
||||
|
||||
Reference in new issue
Block a user