187 lines
7.8 KiB
Python
187 lines
7.8 KiB
Python
"""The prep stage: map what the readers handed on, then fold the copies.
|
|
|
|
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
|
and marks the role due. `run` maps every due role: its company through
|
|
`company_aliases`, its places against the gazetteer, its level, contract
|
|
and remote through the host's tables (`app.prep.words`), its country from
|
|
the places (`app.prep.country`), and a remote role's area
|
|
(`app.prep.area`), and its `deadline` (the host's date, else the one
|
|
`extract` read). `remote` is the model's (`remote_model`), else the
|
|
host's word, else what the place line or a search role's title says. When
|
|
the tables or the gazetteer in code (`places.seed`) changed since the last
|
|
run (`fingerprint`, kept in state as `prep:tables`), the new places and
|
|
spellings are added to the database and every open role is mapped again
|
|
from its stored words, with no new pull. Then the words no table knows are
|
|
listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the
|
|
whole database. `run_roles` maps a few roles outside a run, for `extract`,
|
|
whose model answer changes remote and area.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any, NamedTuple
|
|
|
|
from app import places
|
|
from app.models import company_key
|
|
from app.models import company_fold
|
|
from app.prep import words
|
|
from app.prep.area import area
|
|
from app.prep.country import country, of_host
|
|
from app.prep.hosts import TABLES
|
|
from app.prep.words import Tables
|
|
from app.storage import Store
|
|
|
|
#: Raised when the rules in this package change the way a stored word maps,
|
|
#: so every open role is mapped again as it is after a table changes. 3: the
|
|
#: remote area, and remote as the model's before the host's word. 4: the
|
|
#: deadline, the host's before the model's.
|
|
VERSION = 4
|
|
|
|
#: The state key the last run's `fingerprint` is kept under.
|
|
STATE = "prep:tables"
|
|
|
|
|
|
|
|
class Prepped(NamedTuple):
|
|
"""What one run did: roles read, rows written, whether every role was
|
|
mapped again, words left unmapped, and the open copies after the fold
|
|
(None when it was not run)."""
|
|
|
|
read: int
|
|
written: int
|
|
remapped: bool
|
|
unmapped: int
|
|
copies: int | None
|
|
|
|
|
|
def fingerprint(tables: Tables = TABLES) -> str:
|
|
"""A hash of every host table, the shared remote words, the gazetteer the
|
|
code describes and `VERSION`."""
|
|
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
|
"places": repr(places.seed())}, sort_keys=True)
|
|
return hashlib.sha1(blob.encode()).hexdigest()
|
|
|
|
|
|
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
|
|
"""The key a role from `name` belongs to, as `Store.resolve_company`
|
|
has it: the company an alias names, the name's own key first when
|
|
several share the fold, else `company_key(name)`."""
|
|
own = company_key(name)
|
|
keys = folds.get(company_fold(name), ())
|
|
return own if own in keys or not keys else keys[0]
|
|
|
|
|
|
def _places(raw: Any) -> list[Any]:
|
|
try:
|
|
found = json.loads(raw) if isinstance(raw, str) else raw
|
|
except ValueError:
|
|
return []
|
|
return found if isinstance(found, list) else []
|
|
|
|
|
|
def _texts(row: Mapping[str, Any]) -> list[str]:
|
|
"""The role's place texts: `raw`'s places, else the line cut at `;`."""
|
|
given = _places(row["raw_places"])
|
|
return [words.place_text(p) for p in given] if given \
|
|
else str(row["place"] or "").split(";")
|
|
|
|
|
|
def _model_names(raw: Any) -> list[str] | None:
|
|
"""`remote_where_model` read: None never asked, else the names."""
|
|
if raw is None:
|
|
return None
|
|
return [str(name) for name in _places(raw)]
|
|
|
|
|
|
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
|
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
|
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
|
host = str(row["host"] or "")
|
|
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
|
texts = _texts(row)
|
|
remote = row["remote_model"] \
|
|
or words.remote(host, row["remote_host"] or None, (), "", tables) \
|
|
or words.remote(host, None, _places(row["raw_places"]), row["place"], tables,
|
|
title=row["title"] if row["source"] == "search" else "")
|
|
return {
|
|
"uid": row["uid"],
|
|
"due": row["prep_due"],
|
|
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
|
|
"seniority_stated": stated,
|
|
"seniority_title": titled,
|
|
"employment": words.employment(host, row["employment_host"], tables),
|
|
"remote": remote,
|
|
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
|
if remote == "remote" else ()),
|
|
"country": country(gaz, place_ids, row["country_host"]),
|
|
"deadline": row["deadline_host"] or row["deadline_model"] or None,
|
|
}
|
|
|
|
|
|
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
|
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
|
return bool(row["prep_due"]) or level != row["seniority"] \
|
|
or found["remote"] != row["remote"] \
|
|
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
|
or (found["country"] and found["country"] != row["country"]) \
|
|
or found["deadline"] != row["deadline"] \
|
|
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
|
"seniority_title", "employment"))
|
|
|
|
|
|
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
|
|
tables: Tables = TABLES) -> bool:
|
|
"""Whether no table, vocabulary or gazetteer knows this host word."""
|
|
if field == "country":
|
|
return of_host(gaz, word) is None
|
|
if field == "remote" and word in ("true", "false"):
|
|
return False
|
|
return words.lookup(field, host, word, tables) is None
|
|
|
|
|
|
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
|
dupes: bool = True) -> Prepped:
|
|
"""Map the due roles, or every open one when the tables or the
|
|
gazetteer changed, or every role with `remap`; list the unmapped words;
|
|
fold the copies."""
|
|
mark = fingerprint(tables)
|
|
whole = remap or store.state.get(STATE) != mark
|
|
rows = store.prep_rows("due")
|
|
if whole:
|
|
store.seed_places()
|
|
due = {r["uid"] for r in rows}
|
|
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
|
|
ids = store.resolve_role_places([r["uid"] for r in rows])
|
|
gaz = store.gazetteer()
|
|
folds = store.company_folds()
|
|
written = []
|
|
for row in rows:
|
|
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
|
|
if _changed(row, found):
|
|
written.append(found)
|
|
store.write_prep(written)
|
|
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
|
|
store.replace_unmapped(left)
|
|
if whole:
|
|
store.state[STATE] = mark
|
|
copies = store.rebuild_dupes().copies if dupes else None
|
|
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
|
copies=copies)
|
|
|
|
|
|
def run_roles(store: Store, uids: Sequence[str], tables: Tables = TABLES) -> int:
|
|
"""Map these roles now, due or not, and nothing else: what `extract`
|
|
calls for the roles it answered, so the rescore that follows sees the
|
|
model's remote and area. Returns the rows written."""
|
|
if not uids:
|
|
return 0
|
|
rows = store.prep_rows("all", uids=uids)
|
|
ids = store.resolve_role_places([r["uid"] for r in rows])
|
|
gaz = store.gazetteer()
|
|
folds = store.company_folds()
|
|
return store.write_prep([map_role(r, gaz, folds, ids.get(r["uid"], ()), tables)
|
|
for r in rows])
|