"""The prep stage: map what the readers handed on, then fold the copies. A pull stores the host's own words and place texts (`Store.upsert_role`) and marks the role due. `run` maps every due role: its company through `company_aliases`, its places against the gazetteer, its level, contract and remote through the host's tables (`app.prep.words`), and its country from the places (`app.prep.country`). When the tables changed since the last run (`fingerprint`, kept in state as `prep:tables`), every open role is mapped again from its stored words, with no new pull. Then the words no table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the whole database. """ from __future__ import annotations import hashlib import json from collections.abc import Mapping, Sequence from typing import Any, NamedTuple from app import places from app.models import company_key from app.models import company_fold from app.prep import words from app.prep.country import country, of_host from app.prep.hosts import TABLES from app.prep.words import Tables from app.storage import Store #: Raised when the rules in this package change the way a stored word maps, #: so every open role is mapped again as it is after a table changes. VERSION = 1 #: The state key the last run's `fingerprint` is kept under. STATE = "prep:tables" class Prepped(NamedTuple): """What one run did: roles read, rows written, whether every role was mapped again, words left unmapped, and the open copies after the fold (None when it was not run).""" read: int written: int remapped: bool unmapped: int copies: int | None def fingerprint(tables: Tables = TABLES) -> str: """A hash of every host table, the shared remote words and `VERSION`.""" blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS}, sort_keys=True) return hashlib.sha1(blob.encode()).hexdigest() def company(folds: Mapping[str, Sequence[str]], name: str) -> str: """The key a role from `name` belongs to, as `Store.resolve_company` has it: the company an alias names, the name's own key first when several share the fold, else `company_key(name)`.""" own = company_key(name) keys = folds.get(company_fold(name), ()) return own if own in keys or not keys else keys[0] def _places(raw: Any) -> list[Any]: try: found = json.loads(raw) if isinstance(raw, str) else raw except ValueError: return [] return found if isinstance(found, list) else [] def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]], place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]: """One role as `Store.write_prep` takes it, from a `Store.prep_rows` row.""" host = str(row["host"] or "") stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables) return { "uid": row["uid"], "due": row["prep_due"], "company_key": company(folds, row["company"]) if row["company"] else row["company_key"], "seniority_stated": stated, "seniority_title": titled, "employment": words.employment(host, row["employment_host"], tables), "remote": words.remote(host, row["remote_host"] or None, (), "", tables), "remote_places": words.remote( host, None, _places(row["raw_places"]), row["place"], tables, title=row["title"] if row["source"] == "search" else ""), "country": country(gaz, place_ids, row["country_host"]), } def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool: level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"] return bool(row["prep_due"]) or level != row["seniority"] \ or (found["remote"] and found["remote"] != row["remote"]) \ or (found["remote_places"] and not row["remote"]) \ or (found["country"] and found["country"] != row["country"]) \ or any(found[c] != row[c] for c in ("company_key", "seniority_stated", "seniority_title", "employment")) def unknown(field: str, host: str, word: str, gaz: places.Gazetteer, tables: Tables = TABLES) -> bool: """Whether no table, vocabulary or gazetteer knows this host word.""" if field == "country": return of_host(gaz, word) is None if field == "remote" and word in ("true", "false"): return False return words.lookup(field, host, word, tables) is None def run(store: Store, *, remap: bool = False, tables: Tables = TABLES, dupes: bool = True) -> Prepped: """Map the due roles, or every open one when the tables changed, or every role with `remap`; list the unmapped words; fold the copies.""" mark = fingerprint(tables) whole = remap or store.state.get(STATE) != mark rows = store.prep_rows("due") if whole: due = {r["uid"] for r in rows} rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due] ids = store.resolve_role_places([r["uid"] for r in rows]) gaz = store.gazetteer() folds = store.company_folds() written = [] for row in rows: found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables) if _changed(row, found): written.append(found) store.write_prep(written) left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)] store.replace_unmapped(left) if whole: store.state[STATE] = mark copies = store.rebuild_dupes().copies if dupes else None return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left), copies=copies)