Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
+19
-18
@@ -24,6 +24,7 @@ from app.places import Gazetteer, fold, seed
|
||||
from app.storage import Store
|
||||
|
||||
from test_places import LINES, ORACLE_TOKENS
|
||||
from prepped import upsert
|
||||
|
||||
|
||||
def gazetteer() -> Gazetteer:
|
||||
@@ -173,7 +174,7 @@ def test_a_fresh_file_is_seeded(store: Store) -> None:
|
||||
|
||||
|
||||
def test_a_joined_line_is_several_places(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
|
||||
upsert(store, posting("1", place="Copenhagen; London; Warsaw"))
|
||||
uid = posting("1").uid
|
||||
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
|
||||
(2, "Warsaw", "Warsaw")]
|
||||
@@ -183,7 +184,7 @@ def test_a_joined_line_is_several_places(store: Store) -> None:
|
||||
|
||||
|
||||
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Berlin, Munich", places=[
|
||||
upsert(store, posting("1", place="Berlin, Munich", places=[
|
||||
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
|
||||
{"text": "Remote (DE)", "remote": True}]))
|
||||
assert texts(store, posting("1").uid) == [
|
||||
@@ -191,7 +192,7 @@ def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
||||
|
||||
|
||||
def test_places_without_a_line_make_the_line(store: Store) -> None:
|
||||
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
|
||||
upsert(store, posting("1", places=[{"text": "Lund", "remote": None},
|
||||
{"text": "Malmö", "remote": None}]))
|
||||
role = store.role(posting("1").uid)
|
||||
assert role is not None
|
||||
@@ -199,26 +200,26 @@ def test_places_without_a_line_make_the_line(store: Store) -> None:
|
||||
|
||||
|
||||
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Aarhus"))
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1", place="Aarhus"))
|
||||
upsert(store, posting("1"))
|
||||
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
|
||||
store.upsert_role(posting("1", place="Odense"))
|
||||
upsert(store, posting("1", place="Odense"))
|
||||
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
|
||||
|
||||
|
||||
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Aarhus"))
|
||||
upsert(store, posting("1", place="Aarhus"))
|
||||
uid = posting("1").uid
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
||||
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
|
||||
store.upsert_role(posting("1", place="Aalborg"))
|
||||
upsert(store, posting("1", place="Aalborg"))
|
||||
assert [(p.text, p.url) for p in store.role_places(uid)] == [
|
||||
("Aalborg", ""), ("Odense", "https://copy")]
|
||||
|
||||
|
||||
def test_an_extracted_place_is_resolved(store: Store) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
uid = posting("1").uid
|
||||
store.set_extracted(uid, {"place": "Lund, Sweden"})
|
||||
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
|
||||
@@ -236,7 +237,7 @@ def several(store: Store) -> dict[str, str]:
|
||||
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
|
||||
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
|
||||
for key, line in lines.items():
|
||||
store.upsert_role(posting(key, place=line))
|
||||
upsert(store, posting(key, place=line))
|
||||
return {key: posting(key).uid for key in lines}
|
||||
|
||||
|
||||
@@ -274,7 +275,7 @@ def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
|
||||
|
||||
|
||||
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
|
||||
store.upsert_role(posting("1", country="DK"))
|
||||
upsert(store, posting("1", country="DK"))
|
||||
assert uids(store, "country:DK") == {posting("1").uid}
|
||||
|
||||
|
||||
@@ -312,15 +313,15 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store,
|
||||
|
||||
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
|
||||
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
|
||||
store.upsert_role(posting(str(n), place=line))
|
||||
upsert(store, posting(str(n), place=line))
|
||||
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
|
||||
("jena", "Jena", 1)]
|
||||
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
|
||||
|
||||
|
||||
def test_an_alias_links_what_it_names(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Kbh. K"))
|
||||
store.upsert_role(posting("2", place="Copenhagen"))
|
||||
upsert(store, posting("1", place="Kbh. K"))
|
||||
upsert(store, posting("2", place="Copenhagen"))
|
||||
uid = posting("1").uid
|
||||
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
|
||||
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
|
||||
@@ -333,8 +334,8 @@ def test_an_alias_links_what_it_names(store: Store) -> None:
|
||||
|
||||
|
||||
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Oberkochen; Jena"))
|
||||
store.upsert_role(posting("2", place="Paderborn, Germany"))
|
||||
upsert(store, posting("1", place="Oberkochen; Jena"))
|
||||
upsert(store, posting("2", place="Paderborn, Germany"))
|
||||
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
|
||||
store.add_place("Paderborn", "city", "country:DE")
|
||||
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
|
||||
@@ -355,7 +356,7 @@ def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
|
||||
|
||||
|
||||
def test_relink_places_resolves_every_role_again(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Oberkochen"))
|
||||
upsert(store, posting("1", place="Oberkochen"))
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
conn.execute("UPDATE role_places SET place_id = "
|
||||
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
|
||||
@@ -381,7 +382,7 @@ def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tup
|
||||
lines = {}
|
||||
for n, (place, country) in enumerate(LINES):
|
||||
item = posting(f"line{n}", place=place, country=country)
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
lines[item.uid] = (place, country)
|
||||
return store, lines
|
||||
|
||||
|
||||
Reference in new issue
Block a user