Prep stage: host words stored raw, mapped centrally in app/prep after every pull

upsert_role stores the host's own words (roles.*_host), the place texts
unresolved when they change, and the company key unresolved on insert, and
marks the role prep_due. app.prep maps seniority, employment, remote and
country through the host tables in app/prep/hosts, resolves places and
company aliases, lists unknown words in `unmapped`, remaps every open role
when the tables' fingerprint changes, and runs rebuild_dupes, which moves
out of the pull's settle. Migration 22 adds the columns and the table and
fills the host words from raw. Cascade: pull, prep, embed, score, enrich,
prep, embed, score, extract, score, summarise. `app prep [unmapped]
[--remap]`, /api/prep/unmapped and a fold on the Sources page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:19:22 +02:00
1 parent a66aac95cf
commit d04ceae7e0
31 files changed
+1468 -330

No files matched your search

+19 -16
View File
@@ -24,6 +24,7 @@ from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
@@ -97,7 +98,7 @@ def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
title=title, body=body, published_at=published_at,
raw={"company": company, **raw})
item.source = "ats"
store.upsert_role(item)
upsert(store, item)
return item.uid
@@ -1023,7 +1024,7 @@ def test_the_shipped_starter_searches_validate() -> None:
# ---------------------------------------------------------------- cascade
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
@@ -1053,14 +1054,15 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
assert batch is not None
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
# Enrich found no board with role pages, so the embed and score behind it
# are passed over.
assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"]
# Enrich found no board with role pages, so the prep, embed and score
# behind it are passed over.
assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score",
"summarise"]
work = [j.kind for j in jobs if j.kind != FINAL]
# The `pull` job and the chain it asked for are one cascade, so the Jobs
# tab shows a scheduled pull as one run. Extract and summarise are off and
# find nothing; score runs twice, before and after extract.
assert work == ["pull", "pull_batch", "embed", "score", "score"]
assert work == ["pull", "pull_batch", "prep", "embed", "score", "score"]
assert all(j.state == "done" for j in jobs)
assert len(store.scores(agent.id)) == 1
# The board was stamped with the run the `pull` job claimed, and the
@@ -1109,11 +1111,12 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score",
"extract", "score", "summarise")
assert [cascade.follows("pull", at) for at in range(9)] == \
["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep",
"embed", "score", "extract", "score", "summarise")
assert [cascade.follows("pull", at) for at in range(11)] == \
["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score",
"summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == ""
assert cascade.follows("nope", 0) == ""
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
@@ -1126,10 +1129,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
made.append(t)
return t
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5},
context=Context())
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 8
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 10