Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
@@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any
|
||||
@@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
|
||||
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
||||
|
||||
|
||||
def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None:
|
||||
"""Replace one role's own places with these texts, unresolved: what a
|
||||
pull stores, for `app.prep` to resolve (`relink_uids`)."""
|
||||
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
|
||||
conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None)
|
||||
for position, text in enumerate(place_texts)])
|
||||
|
||||
|
||||
def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve the own places of these roles, writing only those that now
|
||||
resolve differently. Returns `uid -> place ids` in the host's order for
|
||||
every role of `uids` with places, what prep reads the country from."""
|
||||
held: dict[str, dict[tuple[int, str], set[int | None]]] = {}
|
||||
for start in range(0, len(uids), 5000):
|
||||
chunk = json.dumps(list(uids[start:start + 5000]))
|
||||
for uid, position, text, place_id in conn.execute(
|
||||
"SELECT uid, position, text, place_id FROM role_places WHERE url = '' "
|
||||
"AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid",
|
||||
(chunk,)):
|
||||
held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
|
||||
found: dict[str, list[int]] = {}
|
||||
changed = []
|
||||
for uid, rows_of in held.items():
|
||||
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of}
|
||||
if now != rows_of:
|
||||
changed.append((uid, [text for _, text in sorted(rows_of)]))
|
||||
found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])]
|
||||
write_many(conn, gaz, changed)
|
||||
return found
|
||||
|
||||
|
||||
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
||||
"""`write` for many roles in one statement each way. Returns rows written."""
|
||||
|
||||
Reference in new issue
Block a user