Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
+12
-11
@@ -13,6 +13,7 @@ from app.models import company_fold
|
||||
from app.storage import Store
|
||||
from app.storage.schema import _add_company_aliases
|
||||
from desk.models import Item
|
||||
from prepped import upsert
|
||||
|
||||
|
||||
def undo_v20(conn: sqlite3.Connection) -> None:
|
||||
@@ -56,10 +57,10 @@ def test_a_company_is_its_own_alias(store: Store) -> None:
|
||||
|
||||
def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
|
||||
store.add_company("Ørsted")
|
||||
store.upsert_role(posting("1", "Orsted"))
|
||||
store.upsert_role(posting("2", "ORSTED A/S"))
|
||||
store.upsert_role(posting("3", "Ørsted"))
|
||||
store.upsert_role(posting("4", "Nobody"))
|
||||
upsert(store, posting("1", "Orsted"))
|
||||
upsert(store, posting("2", "ORSTED A/S"))
|
||||
upsert(store, posting("3", "Ørsted"))
|
||||
upsert(store, posting("4", "Nobody"))
|
||||
assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"]
|
||||
assert [c.key for c in store.companies()] == ["ørsted"]
|
||||
|
||||
@@ -67,7 +68,7 @@ def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
|
||||
def test_an_added_alias_finds_the_company(store: Store) -> None:
|
||||
store.add_company("Ørsted")
|
||||
store.add_alias("ørsted", "Dong Energy")
|
||||
store.upsert_role(posting("1", "Dong Energy A/S"))
|
||||
upsert(store, posting("1", "Dong Energy A/S"))
|
||||
assert company_of(store, "1") == "ørsted"
|
||||
with pytest.raises(KeyError):
|
||||
store.add_alias("nope", "X")
|
||||
@@ -90,15 +91,15 @@ def test_duplicates_are_listed_by_fold_and_not_merged(store: Store) -> None:
|
||||
assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(store.companies()) == 3
|
||||
# The company with the name's own key wins when a role names it.
|
||||
store.upsert_role(posting("1", "Orsted"))
|
||||
upsert(store, posting("1", "Orsted"))
|
||||
assert company_of(store, "1") == "orsted"
|
||||
|
||||
|
||||
def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
|
||||
store.add_company("Ørsted")
|
||||
store.add_company("Orsted")
|
||||
store.upsert_role(posting("1", "Orsted"))
|
||||
store.upsert_role(posting("2", "Ørsted"))
|
||||
upsert(store, posting("1", "Orsted"))
|
||||
upsert(store, posting("2", "Ørsted"))
|
||||
sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted",
|
||||
board="greenhouse")
|
||||
a, b = store.save_agent("A"), store.save_agent("B")
|
||||
@@ -116,7 +117,7 @@ def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
|
||||
assert store.duplicate_companies() == []
|
||||
assert store.company_aliases() == {"ørsted": ["orsted"]}
|
||||
# A later role from the gone spelling lands on the kept company.
|
||||
store.upsert_role(posting("3", "Orsted"))
|
||||
upsert(store, posting("3", "Orsted"))
|
||||
assert company_of(store, "3") == "ørsted"
|
||||
with pytest.raises(ValueError):
|
||||
store.merge_companies("ørsted", "ørsted")
|
||||
@@ -147,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
||||
"VALUES (?, ?, '', '')", (key, name))
|
||||
conn.execute("PRAGMA user_version = 19")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [20, 21]
|
||||
assert reopened.migrations_applied == [20, 21, 22]
|
||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(reopened.companies()) == 3
|
||||
|
||||
@@ -184,7 +185,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
||||
conn.execute("ALTER TABLE companies DROP COLUMN all_roles")
|
||||
conn.execute("PRAGMA user_version = 20")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [21]
|
||||
assert reopened.migrations_applied == [21, 22]
|
||||
assert {c.key: c.all_roles for c in reopened.companies()} == {
|
||||
"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||
params = {s.target: s.params for s in reopened.subscriptions(source="ats")}
|
||||
|
||||
Reference in new issue
Block a user