Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
+19
-16
@@ -24,6 +24,7 @@ from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
|
||||
from app.jobs import cascade, embed, score # noqa: E402
|
||||
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from prepped import upsert # noqa: E402
|
||||
|
||||
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
||||
|
||||
@@ -97,7 +98,7 @@ def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
|
||||
title=title, body=body, published_at=published_at,
|
||||
raw={"company": company, **raw})
|
||||
item.source = "ats"
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
return item.uid
|
||||
|
||||
|
||||
@@ -1023,7 +1024,7 @@ def test_the_shipped_starter_searches_validate() -> None:
|
||||
|
||||
# ---------------------------------------------------------------- cascade
|
||||
|
||||
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
||||
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
|
||||
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||
@@ -1053,14 +1054,15 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
||||
assert batch is not None
|
||||
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
||||
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
||||
# Enrich found no board with role pages, so the embed and score behind it
|
||||
# are passed over.
|
||||
assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"]
|
||||
# Enrich found no board with role pages, so the prep, embed and score
|
||||
# behind it are passed over.
|
||||
assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score",
|
||||
"summarise"]
|
||||
work = [j.kind for j in jobs if j.kind != FINAL]
|
||||
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
||||
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
||||
# find nothing; score runs twice, before and after extract.
|
||||
assert work == ["pull", "pull_batch", "embed", "score", "score"]
|
||||
assert work == ["pull", "pull_batch", "prep", "embed", "score", "score"]
|
||||
assert all(j.state == "done" for j in jobs)
|
||||
assert len(store.scores(agent.id)) == 1
|
||||
# The board was stamped with the run the `pull` job claimed, and the
|
||||
@@ -1109,11 +1111,12 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
|
||||
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
||||
"""By position, not by name: `score` is in the chain twice, once before
|
||||
extract and once after, and the second must lead on to summarise."""
|
||||
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score",
|
||||
"extract", "score", "summarise")
|
||||
assert [cascade.follows("pull", at) for at in range(9)] == \
|
||||
["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""]
|
||||
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
|
||||
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep",
|
||||
"embed", "score", "extract", "score", "summarise")
|
||||
assert [cascade.follows("pull", at) for at in range(11)] == \
|
||||
["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score",
|
||||
"summarise", ""]
|
||||
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == ""
|
||||
assert cascade.follows("nope", 0) == ""
|
||||
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
||||
|
||||
@@ -1126,10 +1129,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
made.append(t)
|
||||
return t
|
||||
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5},
|
||||
context=Context())
|
||||
assert out == {"task": "score", "next": "extract"}
|
||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6}
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
|
||||
context=Context())
|
||||
assert out["next"] == "summarise" and made[1].carries["at"] == 8
|
||||
assert out == {"task": "score", "next": "extract"}
|
||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8}
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9},
|
||||
context=Context())
|
||||
assert out["next"] == "summarise" and made[1].carries["at"] == 10
|
||||
Reference in new issue
Block a user