Prep stage: host words stored raw, mapped centrally in app/prep after every pull

upsert_role stores the host's own words (roles.*_host), the place texts
unresolved when they change, and the company key unresolved on insert, and
marks the role prep_due. app.prep maps seniority, employment, remote and
country through the host tables in app/prep/hosts, resolves places and
company aliases, lists unknown words in `unmapped`, remaps every open role
when the tables' fingerprint changes, and runs rebuild_dupes, which moves
out of the pull's settle. Migration 22 adds the columns and the table and
fills the host words from raw. Cascade: pull, prep, embed, score, enrich,
prep, embed, score, extract, score, summarise. `app prep [unmapped]
[--remap]`, /api/prep/unmapped and a fold on the Sources page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:19:22 +02:00
1 parent a66aac95cf
commit d04ceae7e0
31 files changed
+1468 -330

No files matched your search

+55 -54
View File
@@ -23,6 +23,7 @@ from app.web.api import _score_after_change
from app.web.pages import SECTIONS
from auth import AuthConfig
from desk.models import Item
from prepped import upsert
class Clock:
@@ -75,9 +76,9 @@ def client(app: FastAPI) -> Iterator[TestClient]:
def test_list_roles_filters_by_query_place_and_function(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering"))
store.upsert_role(posting("2", company="Acme", place="Munich", function="sales"))
store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering"))
upsert(store, posting("1", company="Acme", place="Berlin", function="engineering"))
upsert(store, posting("2", company="Acme", place="Munich", function="sales"))
upsert(store, posting("3", company="Other", place="Berlin", function="engineering"))
uid = {r.source_id: r.uid for r in store.roles()}
res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"})
@@ -100,7 +101,7 @@ def test_list_roles_filters_by_company_keys_places_and_seniority(
("berlin", "Other", "Berlin", "Senior Engineer")):
item = posting(sid, company=company, place=place)
item.title = title
store.upsert_role(item)
upsert(store, item)
uid = {r.source_id: r.uid for r in store.roles()}
def shown(**params: Any) -> set[str]:
@@ -133,7 +134,7 @@ def test_role_facets_count_each_facet_under_the_other_filters(
("c", "Engineer", {"salary_text": "EUR 50 per hour"})):
item = posting(sid, **raw)
item.title = title
store.upsert_role(item)
upsert(store, item)
res = client.get("/api/roles/facets",
params={"seniority": ["senior"], "seniority_unknown": "false",
@@ -152,7 +153,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden(
for sid, company in (("1", "Ørsted"), ("2", "Ørsted"), ("3", "Data Minds"),
("4", "Big Data Corp"), ("5", "Big Data Corp"), ("6", "Big Data Corp"),
("7", "Metadata Ltd"), ("8", "Hidden Data")):
store.upsert_role(posting(sid, company=company))
upsert(store, posting(sid, company=company))
agent = store.save_agent("A")
store.set_affinity(agent.id, "hidden-data", -1)
store.set_affinity(agent.id, "ørsted", 2)
@@ -177,7 +178,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden(
def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None:
for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"),
("4", "København K"), ("5", "Kolding")):
store.upsert_role(posting(sid, place=place))
upsert(store, posting(sid, place=place))
res = client.get("/api/suggest/places", params={"q": "ska"})
assert res.status_code == 200
@@ -200,10 +201,10 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store)
"""`salary_text` and `salary_state` (app/models.py) are on every role the
API serves, and `pay` (`Store.PAY`) filters the feed by it, in SQL before
`limit` like the other filters."""
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
upsert(store, posting("normalised", salary_min=70000, salary_max=90000,
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("none"))
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
upsert(store, posting("none"))
uid = {r.source_id: r.uid for r in store.roles()}
items = {r["uid"]: r for r in client.get("/api/roles").json()["items"]}
@@ -223,7 +224,7 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store)
def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
for i in range(3):
store.upsert_role(posting(str(i)))
upsert(store, posting(str(i)))
assert len(client.get("/api/roles", params={"limit": 1}).json()["items"]) == 1
assert client.get("/api/roles", params={"limit": 0}).status_code == 422
assert client.get("/api/roles", params={"limit": 1001}).status_code == 422
@@ -231,8 +232,8 @@ def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Store) -> None:
store.add_company("Acme", key="acme")
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Other"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Other"))
agent = store.save_agent("A")
other = store.save_agent("B")
uid = {r.source_id: r.uid for r in store.roles()}
@@ -265,10 +266,10 @@ def test_muted_is_excluded_before_the_limit_not_after(
store.clock = clock
store.add_company("Acme", key="acme")
store.set_affinity(store.save_agent("A").id, "acme", -1)
store.upsert_role(posting("other", company="Other"))
upsert(store, posting("other", company="Other"))
clock.tick()
for i in range(3):
store.upsert_role(posting(f"a{i}", company="Acme"))
upsert(store, posting(f"a{i}", company="Acme"))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
@@ -282,7 +283,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
clock = Clock()
store.clock = clock
for i in range(5):
store.upsert_role(posting(str(i)))
upsert(store, posting(str(i)))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
@@ -298,7 +299,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) -> None:
for sid in ("liked", "passed", "fine", "open"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
@@ -340,7 +341,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
def test_votes_are_per_agent_over_the_api(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
one = store.save_agent("One")
two = store.save_agent("Two")
@@ -364,7 +365,7 @@ def test_a_vote_or_an_affinity_needs_an_agent_to_belong_to(
client: TestClient, store: Store) -> None:
"""Nothing to vote as: a 422 with a sentence, not a vote that lands
nowhere. Reading is fine, everything is simply unjudged and neutral."""
store.upsert_role(posting("1"))
upsert(store, posting("1"))
store.add_company("Acme", key="acme")
uid = posting("1").uid
@@ -382,7 +383,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
clock = Clock()
store.clock = clock
for sid in ("1", "2", "3"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
@@ -403,7 +404,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
def test_a_vote_moves_a_role_in_the_ranking_at_once(
client: TestClient, store: Store) -> None:
for sid in ("a", "b", "c"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
label = {v: k for k, v in uid.items()}
agent = store.save_agent("A")
@@ -424,7 +425,7 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once(
def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
res = client.post(f"/api/roles/{uid}/read")
@@ -440,8 +441,8 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S
def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
upsert(store, posting("1"))
upsert(store, posting("2"))
ids = {r.source_id: r.uid for r in store.roles()}
uid = posting("1").uid
client.post(f"/api/roles/{uid}/read")
@@ -455,8 +456,8 @@ def test_list_roles_filters_by_include_read(client: TestClient, store: Store) ->
def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
upsert(store, posting("1"))
upsert(store, posting("2"))
uid = {r.source_id: r.uid for r in store.roles()}
store.set_tags(uid["1"], ["python", "backend"])
store.set_tags(uid["2"], ["python", "frontend"])
@@ -476,8 +477,8 @@ def test_filters_lists_tag_counts_and_respects_show_muted(
store.add_company("Hidden", key="hidden")
agent = store.save_agent("A") # the default agent, whose hidden companies count
store.set_affinity(agent.id, "hidden", -1)
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Hidden"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Hidden"))
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
store.set_tags(uid["1"], ["python"])
store.set_tags(uid["2"], ["python"])
@@ -491,7 +492,7 @@ def test_filters_lists_tag_counts_and_respects_show_muted(
def test_one_role_and_its_read_page(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin"))
upsert(store, posting("1", company="Acme", place="Berlin"))
uid = posting("1").uid
res = client.get(f"/api/roles/{uid}")
@@ -524,7 +525,7 @@ def test_companies_lists_board_and_open_role_count(
client: TestClient, store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme", ats="greenhouse", country="DK")
store.upsert_role(posting("1", company="Acme"), sub)
upsert(store, posting("1", company="Acme"), sub)
store.add_company("Other", key="other", domain="other.example") # no board found
store.add_company("Novo", key="novo", ats="taleo", # followed, not readable
careers_url="https://novo.example/careers")
@@ -561,7 +562,7 @@ def test_judging_a_company_known_only_from_a_search_creates_its_row(
for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")):
item = posting(f"n{i}", company=name)
item.source = "search"
store.upsert_role(item, search)
upsert(store, item, search)
key = "novo-nordisk"
assert store.company(key) is None
@@ -712,7 +713,7 @@ def test_the_watchlist_endpoints_404_for_an_unknown_company(client: TestClient)
def test_save_creates_an_application_with_a_saved_event(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin"))
upsert(store, posting("1", company="Acme", place="Berlin"))
uid = posting("1").uid
res = client.post("/api/applications", json={"role_uid": uid, "status": "saved"})
@@ -744,7 +745,7 @@ def test_save_an_unknown_role_is_refused(client: TestClient) -> None:
def test_set_status_validates_and_writes_an_event(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme"))
upsert(store, posting("1", company="Acme"))
uid = posting("1").uid
saved = client.post("/api/applications", json={"role_uid": uid}).json()
@@ -761,8 +762,8 @@ def test_set_status_validates_and_writes_an_event(
def test_unsave_drops_a_saved_application_and_refuses_one_that_moved_on(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Acme"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Acme"))
first = client.post("/api/applications", json={"role_uid": posting("1").uid}).json()
second = client.post("/api/applications", json={"role_uid": posting("2").uid}).json()
@@ -1069,7 +1070,7 @@ def test_a_duplicate_or_empty_prompt_name_is_a_422_sentence(
def test_the_role_endpoint_serves_the_agents_score_and_its_prompt(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
agent = store.save_agent("A", prompts=[])
store.replace_scores(agent.id, [(uid, 0.8, "", "Forward deployed")])
@@ -1088,7 +1089,7 @@ def test_the_role_endpoint_serves_the_agents_score_and_its_prompt(
def test_deleting_an_agent_takes_its_votes_and_affinities_with_it(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
store.add_company("Acme", key="acme")
uid = posting("1").uid
agent = client.post("/api/agents", json={"name": "A"}).json()
@@ -1127,7 +1128,7 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No
def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
client: TestClient, store: Store) -> None:
for sid in ("low", "high", "gated"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uids = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
store.replace_scores(agent.id, [(uids["low"], 0.2, ""), (uids["high"], 0.8, ""),
@@ -1152,7 +1153,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid",
function="engineering")
item.body = long_description
store.upsert_role(item)
upsert(store, item)
uid = item.uid
row = client.get("/api/roles").json()["items"][0]
@@ -1184,7 +1185,7 @@ def test_list_roles_page_stays_small_even_with_long_postings(
for n in range(60):
item = posting(str(n), company="Acme")
item.body = big
store.upsert_role(item)
upsert(store, item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?",
(big, json.dumps({"blob": big}), big))
@@ -1197,7 +1198,7 @@ def test_list_roles_page_stays_small_even_with_long_postings(
def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
store.set_summary(uid, "A short, hand-written summary.")
row = client.get("/api/roles").json()["items"][0]
@@ -1221,9 +1222,9 @@ def test_sources_lists_searches_and_boards_with_their_countries(
client: TestClient, store: Store) -> None:
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
store.upsert_role(posting("1", company="Acme", place="Copenhagen"), board)
store.upsert_role(posting("2", company="Acme", place="Boston, MA"), board)
store.upsert_role(posting("3", company="Acme", place="Aarhus"), board)
upsert(store, posting("1", company="Acme", place="Copenhagen"), board)
upsert(store, posting("2", company="Acme", place="Boston, MA"), board)
upsert(store, posting("3", company="Acme", place="Aarhus"), board)
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend", "area": 1})
@@ -1244,7 +1245,7 @@ def test_switching_a_board_off_closes_its_roles_and_on_queues_a_pull(
client: TestClient, store: Store, queue: Any) -> None:
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
store.upsert_role(posting("1", company="Acme"), board)
upsert(store, posting("1", company="Acme"), board)
off = client.post(f"/api/sources/{board.id}/switch", json={"on": False}).json()
assert off["closed"] == 1 and off["source"]["on"] is False
@@ -1345,7 +1346,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
body = res.json()
assert (body["found"], body["closed"], body["source"]["target"]) == (0, 0, "greenhouse:acme")
sub_id = body["source"]["id"]
store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id))
upsert(store, posting("1", company="Acme"), store.subscription(sub_id))
[row] = client.get("/api/companies").json()
assert (row["board"], row["subscription_id"], row["ats"], row["ats_slug"]) == (
"greenhouse", sub_id, "greenhouse", "acme")
@@ -1418,7 +1419,7 @@ def test_a_company_from_a_search_only_is_no_longer_one_once_it_has_a_reason(
search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex")
item = posting("1", company="Novo")
item.source = "search"
store.upsert_role(item, search)
upsert(store, item, search)
client.post("/api/companies/novo/affinity", json={"affinity": 1})
[row] = client.get("/api/companies").json()
assert (row["from_search"], row["by_hand"]) == (True, False)
@@ -1514,14 +1515,14 @@ def test_a_company_is_deleted_unless_it_is_read(
).json()["source"]["id"]
res = client.delete("/api/companies/acme")
assert res.status_code == 409 and "greenhouse:acme" in res.json()["detail"]
store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id))
upsert(store, posting("1", company="Acme"), store.subscription(sub_id))
client.delete("/api/companies/acme/board")
assert client.delete("/api/companies/acme").status_code == 204
# Open roles hold a company even where no board is on: they would be
# orphaned from the list.
store.add_company("Beta", key="beta")
store.upsert_role(posting("2", company="Beta"))
upsert(store, posting("2", company="Beta"))
res = client.delete("/api/companies/beta")
assert res.status_code == 409 and "open roles" in res.json()["detail"]
assert store.company("beta") is not None
@@ -1562,7 +1563,7 @@ def test_the_posting_text_is_fetched_once(
from desk.tools import scrape
from desk.tools.article import ArticleContent
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
asked: list[str] = []
@@ -1608,8 +1609,8 @@ def test_summarise_queues_one_role_and_the_job_writes_it(
monkeypatch.setitem(JOBS, "summarise_batch",
replace(JOBS["summarise_batch"], isolate=False, timeout=0.0))
client = TestClient(app)
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
upsert(store, posting("1"))
upsert(store, posting("2"))
uid, bare = posting("1").uid, posting("2").uid
with store.transaction() as conn:
conn.execute("UPDATE roles SET description = '' WHERE uid = ?", (bare,))
@@ -1656,7 +1657,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
board.body = "The whole posting."
search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex")
for item in (board, search, posting("alone", company="Other")):
store.upsert_role(item)
upsert(store, item)
store.rebuild_dupes()
page = client.get("/api/roles").json()
@@ -1675,7 +1676,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
def test_suggest_companies_matches_through_an_alias(client: TestClient, store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(posting("1", company="Orsted"))
upsert(store, posting("1", company="Orsted"))
store.add_alias("ørsted", "Dong Energy")
for q in ("orsted", "ørsted", "dong"):