Settings > Sources > Unmapped shows the place texts no place resolves, with their roles' countries and companies, and the host words no table maps, and copies both as text. Prep fingerprints places.seed(), so a town or spelling added to the code is seeded and every open role is mapped again at the next prep. app prep unlinked lists the places on the command line. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
146 lines
5.9 KiB
Python
146 lines
5.9 KiB
Python
"""The prep stage: map what the readers handed on, then fold the copies.
|
|
|
|
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
|
and marks the role due. `run` maps every due role: its company through
|
|
`company_aliases`, its places against the gazetteer, its level, contract
|
|
and remote through the host's tables (`app.prep.words`), and its country
|
|
from the places (`app.prep.country`). When the tables or the gazetteer in
|
|
code (`places.seed`) changed since the last run (`fingerprint`, kept in
|
|
state as `prep:tables`), the new places and spellings are added to the
|
|
database and every open role is mapped again from its stored words, with
|
|
no new pull. Then the words no
|
|
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
|
|
copies across the whole database.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any, NamedTuple
|
|
|
|
from app import places
|
|
from app.models import company_key
|
|
from app.models import company_fold
|
|
from app.prep import words
|
|
from app.prep.country import country, of_host
|
|
from app.prep.hosts import TABLES
|
|
from app.prep.words import Tables
|
|
from app.storage import Store
|
|
|
|
#: Raised when the rules in this package change the way a stored word maps,
|
|
#: so every open role is mapped again as it is after a table changes.
|
|
VERSION = 1
|
|
|
|
#: The state key the last run's `fingerprint` is kept under.
|
|
STATE = "prep:tables"
|
|
|
|
|
|
|
|
class Prepped(NamedTuple):
|
|
"""What one run did: roles read, rows written, whether every role was
|
|
mapped again, words left unmapped, and the open copies after the fold
|
|
(None when it was not run)."""
|
|
|
|
read: int
|
|
written: int
|
|
remapped: bool
|
|
unmapped: int
|
|
copies: int | None
|
|
|
|
|
|
def fingerprint(tables: Tables = TABLES) -> str:
|
|
"""A hash of every host table, the shared remote words, the gazetteer the
|
|
code describes and `VERSION`."""
|
|
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
|
"places": repr(places.seed())}, sort_keys=True)
|
|
return hashlib.sha1(blob.encode()).hexdigest()
|
|
|
|
|
|
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
|
|
"""The key a role from `name` belongs to, as `Store.resolve_company`
|
|
has it: the company an alias names, the name's own key first when
|
|
several share the fold, else `company_key(name)`."""
|
|
own = company_key(name)
|
|
keys = folds.get(company_fold(name), ())
|
|
return own if own in keys or not keys else keys[0]
|
|
|
|
|
|
def _places(raw: Any) -> list[Any]:
|
|
try:
|
|
found = json.loads(raw) if isinstance(raw, str) else raw
|
|
except ValueError:
|
|
return []
|
|
return found if isinstance(found, list) else []
|
|
|
|
|
|
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
|
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
|
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
|
host = str(row["host"] or "")
|
|
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
|
return {
|
|
"uid": row["uid"],
|
|
"due": row["prep_due"],
|
|
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
|
|
"seniority_stated": stated,
|
|
"seniority_title": titled,
|
|
"employment": words.employment(host, row["employment_host"], tables),
|
|
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
|
|
"remote_places": words.remote(
|
|
host, None, _places(row["raw_places"]), row["place"], tables,
|
|
title=row["title"] if row["source"] == "search" else ""),
|
|
"country": country(gaz, place_ids, row["country_host"]),
|
|
}
|
|
|
|
|
|
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
|
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
|
return bool(row["prep_due"]) or level != row["seniority"] \
|
|
or (found["remote"] and found["remote"] != row["remote"]) \
|
|
or (found["remote_places"] and not row["remote"]) \
|
|
or (found["country"] and found["country"] != row["country"]) \
|
|
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
|
"seniority_title", "employment"))
|
|
|
|
|
|
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
|
|
tables: Tables = TABLES) -> bool:
|
|
"""Whether no table, vocabulary or gazetteer knows this host word."""
|
|
if field == "country":
|
|
return of_host(gaz, word) is None
|
|
if field == "remote" and word in ("true", "false"):
|
|
return False
|
|
return words.lookup(field, host, word, tables) is None
|
|
|
|
|
|
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
|
dupes: bool = True) -> Prepped:
|
|
"""Map the due roles, or every open one when the tables or the
|
|
gazetteer changed, or every role with `remap`; list the unmapped words;
|
|
fold the copies."""
|
|
mark = fingerprint(tables)
|
|
whole = remap or store.state.get(STATE) != mark
|
|
rows = store.prep_rows("due")
|
|
if whole:
|
|
store.seed_places()
|
|
due = {r["uid"] for r in rows}
|
|
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
|
|
ids = store.resolve_role_places([r["uid"] for r in rows])
|
|
gaz = store.gazetteer()
|
|
folds = store.company_folds()
|
|
written = []
|
|
for row in rows:
|
|
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
|
|
if _changed(row, found):
|
|
written.append(found)
|
|
store.write_prep(written)
|
|
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
|
|
store.replace_unmapped(left)
|
|
if whole:
|
|
store.state[STATE] = mark
|
|
copies = store.rebuild_dupes().copies if dupes else None
|
|
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
|
copies=copies)
|