Files
jobboard/app/prep/stage.py
T

187 lines
7.8 KiB
Python

"""The prep stage: map what the readers handed on, then fold the copies.
A pull stores the host's own words and place texts (`Store.upsert_role`)
and marks the role due. `run` maps every due role: its company through
`company_aliases`, its places against the gazetteer, its level, contract
and remote through the host's tables (`app.prep.words`), its country from
the places (`app.prep.country`), and a remote role's area
(`app.prep.area`), and its `deadline` (the host's date, else the one
`extract` read). `remote` is the model's (`remote_model`), else the
host's word, else what the place line or a search role's title says. When
the tables or the gazetteer in code (`places.seed`) changed since the last
run (`fingerprint`, kept in state as `prep:tables`), the new places and
spellings are added to the database and every open role is mapped again
from its stored words, with no new pull. Then the words no table knows are
listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the
whole database. `run_roles` maps a few roles outside a run, for `extract`,
whose model answer changes remote and area.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, Sequence
from typing import Any, NamedTuple
from app import places
from app.models import company_key
from app.models import company_fold
from app.prep import words
from app.prep.area import area
from app.prep.country import country, of_host
from app.prep.hosts import TABLES
from app.prep.words import Tables
from app.storage import Store
#: Raised when the rules in this package change the way a stored word maps,
#: so every open role is mapped again as it is after a table changes. 3: the
#: remote area, and remote as the model's before the host's word. 4: the
#: deadline, the host's before the model's.
VERSION = 4
#: The state key the last run's `fingerprint` is kept under.
STATE = "prep:tables"
class Prepped(NamedTuple):
"""What one run did: roles read, rows written, whether every role was
mapped again, words left unmapped, and the open copies after the fold
(None when it was not run)."""
read: int
written: int
remapped: bool
unmapped: int
copies: int | None
def fingerprint(tables: Tables = TABLES) -> str:
"""A hash of every host table, the shared remote words, the gazetteer the
code describes and `VERSION`."""
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
"places": repr(places.seed())}, sort_keys=True)
return hashlib.sha1(blob.encode()).hexdigest()
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
"""The key a role from `name` belongs to, as `Store.resolve_company`
has it: the company an alias names, the name's own key first when
several share the fold, else `company_key(name)`."""
own = company_key(name)
keys = folds.get(company_fold(name), ())
return own if own in keys or not keys else keys[0]
def _places(raw: Any) -> list[Any]:
try:
found = json.loads(raw) if isinstance(raw, str) else raw
except ValueError:
return []
return found if isinstance(found, list) else []
def _texts(row: Mapping[str, Any]) -> list[str]:
"""The role's place texts: `raw`'s places, else the line cut at `;`."""
given = _places(row["raw_places"])
return [words.place_text(p) for p in given] if given \
else str(row["place"] or "").split(";")
def _model_names(raw: Any) -> list[str] | None:
"""`remote_where_model` read: None never asked, else the names."""
if raw is None:
return None
return [str(name) for name in _places(raw)]
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
host = str(row["host"] or "")
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
texts = _texts(row)
remote = row["remote_model"] \
or words.remote(host, row["remote_host"] or None, (), "", tables) \
or words.remote(host, None, _places(row["raw_places"]), row["place"], tables,
title=row["title"] if row["source"] == "search" else "")
return {
"uid": row["uid"],
"due": row["prep_due"],
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
"seniority_stated": stated,
"seniority_title": titled,
"employment": words.employment(host, row["employment_host"], tables),
"remote": remote,
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
if remote == "remote" else ()),
"country": country(gaz, place_ids, row["country_host"]),
"deadline": row["deadline_host"] or row["deadline_model"] or None,
}
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
return bool(row["prep_due"]) or level != row["seniority"] \
or found["remote"] != row["remote"] \
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
or (found["country"] and found["country"] != row["country"]) \
or found["deadline"] != row["deadline"] \
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
"seniority_title", "employment"))
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
tables: Tables = TABLES) -> bool:
"""Whether no table, vocabulary or gazetteer knows this host word."""
if field == "country":
return of_host(gaz, word) is None
if field == "remote" and word in ("true", "false"):
return False
return words.lookup(field, host, word, tables) is None
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
dupes: bool = True) -> Prepped:
"""Map the due roles, or every open one when the tables or the
gazetteer changed, or every role with `remap`; list the unmapped words;
fold the copies."""
mark = fingerprint(tables)
whole = remap or store.state.get(STATE) != mark
rows = store.prep_rows("due")
if whole:
store.seed_places()
due = {r["uid"] for r in rows}
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
ids = store.resolve_role_places([r["uid"] for r in rows])
gaz = store.gazetteer()
folds = store.company_folds()
written = []
for row in rows:
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
if _changed(row, found):
written.append(found)
store.write_prep(written)
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
store.replace_unmapped(left)
if whole:
store.state[STATE] = mark
copies = store.rebuild_dupes().copies if dupes else None
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
copies=copies)
def run_roles(store: Store, uids: Sequence[str], tables: Tables = TABLES) -> int:
"""Map these roles now, due or not, and nothing else: what `extract`
calls for the roles it answered, so the rescore that follows sees the
model's remote and area. Returns the rows written."""
if not uids:
return 0
rows = store.prep_rows("all", uids=uids)
ids = store.resolve_role_places([r["uid"] for r in rows])
gaz = store.gazetteer()
folds = store.company_folds()
return store.write_prep([map_role(r, gaz, folds, ids.get(r["uid"], ()), tables)
for r in rows])