Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
@@ -20,6 +20,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from app import prep # noqa: E402
|
||||
from plugins import readers, search # noqa: E402
|
||||
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
||||
from plugins.search import SearchSource # noqa: E402
|
||||
@@ -28,6 +29,8 @@ from app.jobs import pull # noqa: E402
|
||||
from pipeline import Tally # noqa: E402
|
||||
from app.models import country_of, remote_of # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from prepped import upsert # noqa: E402
|
||||
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||
|
||||
@@ -108,9 +111,9 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) ->
|
||||
it.source = "ats"
|
||||
return it
|
||||
|
||||
store.upsert_role(item("a", place="Copenhagen, Denmark"))
|
||||
store.upsert_role(item("b", place="Remote", country="EU"))
|
||||
store.upsert_role(item("c", place="Somewhere"))
|
||||
upsert(store, item("a", place="Copenhagen, Denmark"))
|
||||
upsert(store, item("b", place="Remote", country="EU"))
|
||||
upsert(store, item("c", place="Somewhere"))
|
||||
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
|
||||
|
||||
|
||||
@@ -281,7 +284,9 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo
|
||||
def _pull(store: Store) -> dict[str, Any]:
|
||||
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
prep.run(store)
|
||||
return closed
|
||||
|
||||
|
||||
def test_a_complete_search_closes_what_it_stopped_listing(
|
||||
|
||||
Reference in new issue
Block a user