Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
+31
-1
@@ -8,6 +8,7 @@
|
||||
app agents list|prompt
|
||||
app stages [extract|summarise on|off]
|
||||
app dupes
|
||||
app prep [unmapped] [--remap]
|
||||
app db init
|
||||
|
||||
`app job` queues the work on a queue of its own and drains it here, so a
|
||||
@@ -160,6 +161,13 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
||||
|
||||
prep = sub.add_parser(
|
||||
"prep", help="map the roles' host words and fold the copies, or list the unmapped words")
|
||||
prep.add_argument("action", nargs="?", choices=["unmapped"],
|
||||
help="list the host words no table maps, instead of running")
|
||||
prep.add_argument("--remap", action="store_true",
|
||||
help="map every role again, closed ones included")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
return parser
|
||||
@@ -188,6 +196,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return run_seniority(args)
|
||||
if args.command == "dupes":
|
||||
return run_dupes(args)
|
||||
if args.command == "prep":
|
||||
return run_prep(args)
|
||||
return init_db(args)
|
||||
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
@@ -655,7 +665,7 @@ def run_seniority(args: argparse.Namespace) -> int:
|
||||
|
||||
|
||||
def run_dupes(args: argparse.Namespace) -> int:
|
||||
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
|
||||
"""`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`,
|
||||
now. Prints the groups (kept roles with a copy), the copies the feed
|
||||
hides and the rows whose `dup_of` changed, as JSON."""
|
||||
from app.storage import Store
|
||||
@@ -666,6 +676,26 @@ def run_dupes(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def run_prep(args: argparse.Namespace) -> int:
|
||||
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
|
||||
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
|
||||
host words no table maps, per field and host, most roles first."""
|
||||
from app import prep
|
||||
from app.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
if args.action == "unmapped":
|
||||
found = store.unmapped()
|
||||
if not found:
|
||||
print("every host word is mapped")
|
||||
for w in found:
|
||||
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
|
||||
return 0
|
||||
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
|
||||
return 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from app.storage import LATEST, Store
|
||||
|
||||
Reference in new issue
Block a user