"""The prep stage: map what the readers handed on, then fold the copies. A pull stores the host's own words and place texts (`Store.upsert_role`) and marks the role due. `run` maps every due role: its company through `company_aliases`, its places against the gazetteer, its level, contract and remote through the host's tables (`app.prep.words`), its country from the places (`app.prep.country`), and a remote role's area (`app.prep.area`), and its `deadline` (the host's date, else the one `extract` read). `remote` is the model's (`remote_model`), else the host's word, else what the place line or a search role's title says. When the tables or the gazetteer in code (`places.seed`) changed since the last run (`fingerprint`, kept in state as `prep:tables`), the new places and spellings are added to the database and every open role is mapped again from its stored words, with no new pull. Then the words no table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the whole database. `run_roles` maps a few roles outside a run, for `extract`, whose model answer changes remote and area. """ from __future__ import annotations import hashlib import json from collections.abc import Mapping, Sequence from typing import Any, NamedTuple from app import places from app.models import company_key from app.models import company_fold from app.prep import words from app.prep.area import area from app.prep.country import country, of_host from app.prep.hosts import TABLES from app.prep.words import Tables from app.storage import Store #: Raised when the rules in this package change the way a stored word maps, #: so every open role is mapped again as it is after a table changes. 3: the #: remote area, and remote as the model's before the host's word. 4: the #: deadline, the host's before the model's. VERSION = 4 #: The state key the last run's `fingerprint` is kept under. STATE = "prep:tables" class Prepped(NamedTuple): """What one run did: roles read, rows written, whether every role was mapped again, words left unmapped, and the open copies after the fold (None when it was not run).""" read: int written: int remapped: bool unmapped: int copies: int | None def fingerprint(tables: Tables = TABLES) -> str: """A hash of every host table, the shared remote words, the gazetteer the code describes and `VERSION`.""" blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS, "places": repr(places.seed())}, sort_keys=True) return hashlib.sha1(blob.encode()).hexdigest() def company(folds: Mapping[str, Sequence[str]], name: str) -> str: """The key a role from `name` belongs to, as `Store.resolve_company` has it: the company an alias names, the name's own key first when several share the fold, else `company_key(name)`.""" own = company_key(name) keys = folds.get(company_fold(name), ()) return own if own in keys or not keys else keys[0] def _places(raw: Any) -> list[Any]: try: found = json.loads(raw) if isinstance(raw, str) else raw except ValueError: return [] return found if isinstance(found, list) else [] def _texts(row: Mapping[str, Any]) -> list[str]: """The role's place texts: `raw`'s places, else the line cut at `;`.""" given = _places(row["raw_places"]) return [words.place_text(p) for p in given] if given \ else str(row["place"] or "").split(";") def _model_names(raw: Any) -> list[str] | None: """`remote_where_model` read: None never asked, else the names.""" if raw is None: return None return [str(name) for name in _places(raw)] def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]], place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]: """One role as `Store.write_prep` takes it, from a `Store.prep_rows` row.""" host = str(row["host"] or "") stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables) texts = _texts(row) remote = row["remote_model"] \ or words.remote(host, row["remote_host"] or None, (), "", tables) \ or words.remote(host, None, _places(row["raw_places"]), row["place"], tables, title=row["title"] if row["source"] == "search" else "") return { "uid": row["uid"], "due": row["prep_due"], "company_key": company(folds, row["company"]) if row["company"] else row["company_key"], "seniority_stated": stated, "seniority_title": titled, "employment": words.employment(host, row["employment_host"], tables), "remote": remote, "remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"])) if remote == "remote" else ()), "country": country(gaz, place_ids, row["country_host"]), "deadline": row["deadline_host"] or row["deadline_model"] or None, } def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool: level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"] return bool(row["prep_due"]) or level != row["seniority"] \ or found["remote"] != row["remote"] \ or found["remote_area"] != json.loads(row["remote_area"] or "[]") \ or (found["country"] and found["country"] != row["country"]) \ or found["deadline"] != row["deadline"] \ or any(found[c] != row[c] for c in ("company_key", "seniority_stated", "seniority_title", "employment")) def unknown(field: str, host: str, word: str, gaz: places.Gazetteer, tables: Tables = TABLES) -> bool: """Whether no table, vocabulary or gazetteer knows this host word.""" if field == "country": return of_host(gaz, word) is None if field == "remote" and word in ("true", "false"): return False return words.lookup(field, host, word, tables) is None def run(store: Store, *, remap: bool = False, tables: Tables = TABLES, dupes: bool = True) -> Prepped: """Map the due roles, or every open one when the tables or the gazetteer changed, or every role with `remap`; list the unmapped words; fold the copies.""" mark = fingerprint(tables) whole = remap or store.state.get(STATE) != mark rows = store.prep_rows("due") if whole: store.seed_places() due = {r["uid"] for r in rows} rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due] ids = store.resolve_role_places([r["uid"] for r in rows]) gaz = store.gazetteer() folds = store.company_folds() written = [] for row in rows: found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables) if _changed(row, found): written.append(found) store.write_prep(written) left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)] store.replace_unmapped(left) if whole: store.state[STATE] = mark copies = store.rebuild_dupes().copies if dupes else None return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left), copies=copies) def run_roles(store: Store, uids: Sequence[str], tables: Tables = TABLES) -> int: """Map these roles now, due or not, and nothing else: what `extract` calls for the roles it answered, so the rescore that follows sees the model's remote and area. Returns the rows written.""" if not uids: return 0 rows = store.prep_rows("all", uids=uids) ids = store.resolve_role_places([r["uid"] for r in rows]) gaz = store.gazetteer() folds = store.company_folds() return store.write_prep([map_role(r, gaz, folds, ids.get(r["uid"], ()), tables) for r in rows])