Prep stage: host words stored raw, mapped centrally in app/prep after every pull

upsert_role stores the host's own words (roles.*_host), the place texts
unresolved when they change, and the company key unresolved on insert, and
marks the role prep_due. app.prep maps seniority, employment, remote and
country through the host tables in app/prep/hosts, resolves places and
company aliases, lists unknown words in `unmapped`, remaps every open role
when the tables' fingerprint changes, and runs rebuild_dupes, which moves
out of the pull's settle. Migration 22 adds the columns and the table and
fills the host words from raw. Cascade: pull, prep, embed, score, enrich,
prep, embed, score, extract, score, summarise. `app prep [unmapped]
[--remap]`, /api/prep/unmapped and a fold on the Sources page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:19:22 +02:00
1 parent a66aac95cf
commit d04ceae7e0
31 files changed
+1468 -330

No files matched your search

+9 -4
View File
@@ -20,6 +20,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from app import prep # noqa: E402
from plugins import readers, search # noqa: E402
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402
@@ -28,6 +29,8 @@ from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
@@ -108,9 +111,9 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) ->
it.source = "ats"
return it
store.upsert_role(item("a", place="Copenhagen, Denmark"))
store.upsert_role(item("b", place="Remote", country="EU"))
store.upsert_role(item("c", place="Somewhere"))
upsert(store, item("a", place="Copenhagen, Denmark"))
upsert(store, item("b", place="Remote", country="EU"))
upsert(store, item("c", place="Somewhere"))
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
@@ -281,7 +284,9 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo
def _pull(store: Store) -> dict[str, Any]:
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
prep.run(store)
return closed
def test_a_complete_search_closes_what_it_stopped_listing(