Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
@@ -0,0 +1,22 @@
|
||||
"""Prep: every decision about a role that needs a rule or data across roles.
|
||||
|
||||
Readers scrape and hand on the host's own words (`Store.upsert_role` keeps
|
||||
them in `roles.*_host`); prep maps them, once, centrally, between the pull
|
||||
and embedding. One door: `from app.prep import ...`.
|
||||
|
||||
words.py seniority, employment and remote: the host's table, the
|
||||
vocabulary itself, the shared remote words
|
||||
country.py the country from the resolved places, else the host's
|
||||
hosts/ one table file per reader (`hosts.TABLES`)
|
||||
stage.py `run`: the stage over the due roles, `unmapped` and the fold
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.prep.country import country
|
||||
from app.prep.hosts import TABLES
|
||||
from app.prep.stage import STATE, Prepped, company, fingerprint, run
|
||||
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
|
||||
|
||||
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
|
||||
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
|
||||
@@ -0,0 +1,55 @@
|
||||
"""A role's country, from its places, else from the one its host states."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from app import places
|
||||
|
||||
|
||||
def of_host(gaz: places.Gazetteer, stated: str) -> str | None:
|
||||
"""The ISO code a host's country names, in the host's spelling: a code
|
||||
("DK", "dk"), or a name the gazetteer has as an alias of a country
|
||||
("Denmark", "Danmark"), `EU` or Europe by name, or a text that resolves to places of one
|
||||
country. '' for nothing stated, None for a text naming no country."""
|
||||
text = (stated or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
if ("country", text.upper()) in gaz.by_key or text.upper() == "EU":
|
||||
return text.upper()
|
||||
named = {"EU" if gaz.places[p].key == "europe" else gaz.places[p].key
|
||||
for p in gaz.aliases.get(places.fold(text), ())
|
||||
if gaz.places[p].kind == "country"
|
||||
or (gaz.places[p].kind, gaz.places[p].key) == ("continent", "europe")}
|
||||
if len(named) == 1:
|
||||
return named.pop()
|
||||
found = {gaz.country(p) for p in gaz.resolve(text)} - {""}
|
||||
return found.pop() if len(found) == 1 else None
|
||||
|
||||
|
||||
def _in_europe(gaz: places.Gazetteer, code: str) -> bool:
|
||||
if code == "EU":
|
||||
return True
|
||||
europe = gaz.by_key.get(("continent", "europe"))
|
||||
place_id = gaz.by_key.get(("country", code))
|
||||
return europe is not None and place_id is not None and europe in gaz.ancestors(place_id)
|
||||
|
||||
|
||||
def country(gaz: places.Gazetteer, place_ids: Iterable[int], host_country: str = "") -> str:
|
||||
"""The one country the role's resolved places lie in, continents aside;
|
||||
Europe alone is `EU`; else, when they
|
||||
lie in several all in Europe, `EU` (what `country_of` wrote for "Dublin,
|
||||
London" and the country gate reads as Europe); else the host's stated
|
||||
country (`of_host`); else ''."""
|
||||
ids = list(place_ids)
|
||||
# A continent beside a country is the wider unit a line adds ("Home
|
||||
# based - EMEA; London" is GB), as `country_of` had it; alone, Europe
|
||||
# is `EU`.
|
||||
found = {gaz.country(p) for p in ids} - {""}
|
||||
if not found and any(gaz.places[p].key == "europe" for p in ids if p in gaz.places):
|
||||
found = {"EU"}
|
||||
if len(found) == 1:
|
||||
return found.pop()
|
||||
if len(found) > 1 and all(_in_europe(gaz, code) for code in found):
|
||||
return "EU"
|
||||
return of_host(gaz, host_country) or ""
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Each host's own words, one module per reader, named as the reader is.
|
||||
|
||||
A module defines any of three dicts, keyed by the host's word lowercased:
|
||||
|
||||
- `SENIORITY`, values in `app.models.SENIORITIES`
|
||||
- `EMPLOYMENT`, values in `app.models.EMPLOYMENTS`
|
||||
- `REMOTE`, values in `app.models.REMOTE` (remote, hybrid, onsite)
|
||||
|
||||
A value of "" is a word the host uses that names no value ("Experienced",
|
||||
"Permanent"): known, so it is not listed as unmapped.
|
||||
|
||||
A host whose employment word is a code of several parts (`permanent_full_time`,
|
||||
`SALARIED_FT`) may add `MATCH`, `[(fragment, value), ...]`, tried in order
|
||||
after `EMPLOYMENT` with the first hit winning, and must then say how a
|
||||
fragment matches in `MATCH_MODE`: "substring" (the fragment is in the
|
||||
lowercased word) or "token" (it equals one of the word's parts split at `_`
|
||||
and `-`, so `ft` does not match inside `software`). `TABLES` loads every
|
||||
module of this package by name, so a table is added by adding its file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
from typing import Any
|
||||
|
||||
#: The dict a module names, by the field it maps.
|
||||
FIELDS = {"seniority": "SENIORITY", "employment": "EMPLOYMENT", "remote": "REMOTE"}
|
||||
|
||||
#: How a `MATCH` fragment may match.
|
||||
MATCH_MODES = ("substring", "token")
|
||||
|
||||
|
||||
def load() -> dict[str, dict[str, Any]]:
|
||||
"""`host -> field -> {word: value}` for every module of the package, the
|
||||
words lowercased and stripped. A host without a table for a field has
|
||||
no entry for it. A `MATCH` is `host -> "match" -> {"mode": ..., "rules":
|
||||
[[fragment, value], ...]}`; a module with `MATCH` and no valid
|
||||
`MATCH_MODE` fails the import by name."""
|
||||
tables: dict[str, dict[str, Any]] = {}
|
||||
for info in sorted(pkgutil.iter_modules(__path__), key=lambda i: i.name):
|
||||
if info.name.startswith("_"):
|
||||
continue
|
||||
module = importlib.import_module(f"{__name__}.{info.name}")
|
||||
for field, name in FIELDS.items():
|
||||
table = getattr(module, name, None)
|
||||
if isinstance(table, dict):
|
||||
tables.setdefault(info.name, {})[field] = {
|
||||
str(k).strip().lower(): str(v) for k, v in table.items()}
|
||||
rules = getattr(module, "MATCH", None)
|
||||
if rules is not None:
|
||||
mode = getattr(module, "MATCH_MODE", None)
|
||||
if mode not in MATCH_MODES:
|
||||
raise ValueError(f"app/prep/hosts/{info.name}.py has MATCH and its MATCH_MODE "
|
||||
f"is {mode!r}, not one of {', '.join(MATCH_MODES)}")
|
||||
tables.setdefault(info.name, {})["match"] = {
|
||||
"mode": mode, "rules": [[str(f).lower(), str(v)] for f, v in rules]}
|
||||
return tables
|
||||
|
||||
|
||||
TABLES = load()
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Platsbanken's words, for prep."""
|
||||
|
||||
#: `workplace_model.label`, in Swedish; mostly empty. Only "Arbete på plats"
|
||||
#: is in the fixtures; the others are JobTech's other labels as the site
|
||||
#: writes them, and a label not here is listed as unmapped.
|
||||
REMOTE: dict[str, str] = {
|
||||
"arbete på plats": "onsite", "på plats": "onsite",
|
||||
"hybridarbete": "hybrid", "hybrid": "hybrid", "delvis på distans": "hybrid",
|
||||
"delvis distans": "hybrid",
|
||||
"distansarbete": "remote", "arbete på distans": "remote", "helt på distans": "remote",
|
||||
"distans": "remote",
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
"""The prep stage: map what the readers handed on, then fold the copies.
|
||||
|
||||
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
||||
and marks the role due. `run` maps every due role: its company through
|
||||
`company_aliases`, its places against the gazetteer, its level, contract
|
||||
and remote through the host's tables (`app.prep.words`), and its country
|
||||
from the places (`app.prep.country`). When the tables changed since the
|
||||
last run (`fingerprint`, kept in state as `prep:tables`), every open role
|
||||
is mapped again from its stored words, with no new pull. Then the words no
|
||||
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
|
||||
copies across the whole database.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from app import places
|
||||
from app.models import company_key
|
||||
from app.models import company_fold
|
||||
from app.prep import words
|
||||
from app.prep.country import country, of_host
|
||||
from app.prep.hosts import TABLES
|
||||
from app.prep.words import Tables
|
||||
from app.storage import Store
|
||||
|
||||
#: Raised when the rules in this package change the way a stored word maps,
|
||||
#: so every open role is mapped again as it is after a table changes.
|
||||
VERSION = 1
|
||||
|
||||
#: The state key the last run's `fingerprint` is kept under.
|
||||
STATE = "prep:tables"
|
||||
|
||||
|
||||
|
||||
class Prepped(NamedTuple):
|
||||
"""What one run did: roles read, rows written, whether every role was
|
||||
mapped again, words left unmapped, and the open copies after the fold
|
||||
(None when it was not run)."""
|
||||
|
||||
read: int
|
||||
written: int
|
||||
remapped: bool
|
||||
unmapped: int
|
||||
copies: int | None
|
||||
|
||||
|
||||
def fingerprint(tables: Tables = TABLES) -> str:
|
||||
"""A hash of every host table, the shared remote words and `VERSION`."""
|
||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
|
||||
sort_keys=True)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()
|
||||
|
||||
|
||||
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
|
||||
"""The key a role from `name` belongs to, as `Store.resolve_company`
|
||||
has it: the company an alias names, the name's own key first when
|
||||
several share the fold, else `company_key(name)`."""
|
||||
own = company_key(name)
|
||||
keys = folds.get(company_fold(name), ())
|
||||
return own if own in keys or not keys else keys[0]
|
||||
|
||||
|
||||
def _places(raw: Any) -> list[Any]:
|
||||
try:
|
||||
found = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except ValueError:
|
||||
return []
|
||||
return found if isinstance(found, list) else []
|
||||
|
||||
|
||||
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
||||
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
||||
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
||||
host = str(row["host"] or "")
|
||||
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
||||
return {
|
||||
"uid": row["uid"],
|
||||
"due": row["prep_due"],
|
||||
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
|
||||
"seniority_stated": stated,
|
||||
"seniority_title": titled,
|
||||
"employment": words.employment(host, row["employment_host"], tables),
|
||||
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
|
||||
"remote_places": words.remote(host, None, _places(row["raw_places"]), row["place"],
|
||||
tables),
|
||||
"country": country(gaz, place_ids, row["country_host"]),
|
||||
}
|
||||
|
||||
|
||||
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||
or (found["remote"] and found["remote"] != row["remote"]) \
|
||||
or (found["remote_places"] and not row["remote"]) \
|
||||
or (found["country"] and found["country"] != row["country"]) \
|
||||
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
||||
"seniority_title", "employment"))
|
||||
|
||||
|
||||
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
|
||||
tables: Tables = TABLES) -> bool:
|
||||
"""Whether no table, vocabulary or gazetteer knows this host word."""
|
||||
if field == "country":
|
||||
return of_host(gaz, word) is None
|
||||
if field == "remote" and word in ("true", "false"):
|
||||
return False
|
||||
return words.lookup(field, host, word, tables) is None
|
||||
|
||||
|
||||
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
||||
dupes: bool = True) -> Prepped:
|
||||
"""Map the due roles, or every open one when the tables changed, or
|
||||
every role with `remap`; list the unmapped words; fold the copies."""
|
||||
mark = fingerprint(tables)
|
||||
whole = remap or store.state.get(STATE) != mark
|
||||
rows = store.prep_rows("due")
|
||||
if whole:
|
||||
due = {r["uid"] for r in rows}
|
||||
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
|
||||
ids = store.resolve_role_places([r["uid"] for r in rows])
|
||||
gaz = store.gazetteer()
|
||||
folds = store.company_folds()
|
||||
written = []
|
||||
for row in rows:
|
||||
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
|
||||
if _changed(row, found):
|
||||
written.append(found)
|
||||
store.write_prep(written)
|
||||
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
|
||||
store.replace_unmapped(left)
|
||||
if whole:
|
||||
store.state[STATE] = mark
|
||||
copies = store.rebuild_dupes().copies if dupes else None
|
||||
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
||||
copies=copies)
|
||||
@@ -0,0 +1,110 @@
|
||||
"""A host's word to the app's vocabulary: seniority, employment and remote.
|
||||
|
||||
Each field is looked up the same way: the host's table in `hosts`, keyed by
|
||||
the word lowercased, then a word that already is a value of the vocabulary
|
||||
(what readers stored before they handed on the host's words, and what a
|
||||
host with the app's own words sends), then, for remote, the shared word map.
|
||||
An employment word may also match a host's `MATCH` fragments. A word none
|
||||
of these knows is unmapped: it maps to '' and `lookup` says so,
|
||||
so the prep stage can list it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of
|
||||
from app.prep.hosts import TABLES
|
||||
from app.seniority import from_title
|
||||
|
||||
#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`).
|
||||
Tables = Mapping[str, Mapping[str, Any]]
|
||||
|
||||
VOCABULARIES: dict[str, tuple[str, ...]] = {
|
||||
"seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE}
|
||||
|
||||
#: How hosts without a table of their own write remote, after `_squash`:
|
||||
#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office").
|
||||
REMOTE_WORDS: dict[str, str] = {
|
||||
"remote": "remote", "fullyremote": "remote", "remotefirst": "remote",
|
||||
"hybrid": "hybrid",
|
||||
"onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite",
|
||||
"officebased": "onsite", "onpremise": "onsite",
|
||||
}
|
||||
|
||||
|
||||
def _match(rules: Mapping[str, Any], key: str) -> str | None:
|
||||
"""The value of the first `MATCH` fragment `key` holds, by its mode."""
|
||||
parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set()
|
||||
for fragment, value in rules["rules"]:
|
||||
if fragment in parts if rules["mode"] == "token" else fragment in key:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def _squash(word: str) -> str:
|
||||
return "".join(c for c in word.lower() if c not in " -_")
|
||||
|
||||
|
||||
def lookup(field: str, host: str, word: str,
|
||||
tables: Tables = TABLES) -> str | None:
|
||||
"""The value `word` maps to for `host`, '' for a known word that names
|
||||
none, None for a word nothing knows. An empty word is ''."""
|
||||
key = (word or "").strip().lower()
|
||||
if not key:
|
||||
return ""
|
||||
table = tables.get(host, {})
|
||||
found = table.get(field, {}).get(key)
|
||||
if found is None and field == "employment" and "match" in table:
|
||||
found = _match(table["match"], key)
|
||||
if found is not None:
|
||||
return found
|
||||
if key in VOCABULARIES[field]:
|
||||
return key
|
||||
if field == "remote":
|
||||
return REMOTE_WORDS.get(_squash(key))
|
||||
return None
|
||||
|
||||
|
||||
def seniority(host: str, word: str, title: str,
|
||||
tables: Tables = TABLES) -> tuple[str, str]:
|
||||
"""`(stated, from the title)`: the host's word through its table, and
|
||||
`seniority.from_title`. `roles.seniority` is the model's level, else the
|
||||
stated one, else the title's (`Store.write_prep`)."""
|
||||
return lookup("seniority", host, word, tables) or "", from_title(title)
|
||||
|
||||
|
||||
def employment(host: str, word: str,
|
||||
tables: Tables = TABLES) -> str:
|
||||
return lookup("employment", host, word, tables) or ""
|
||||
|
||||
|
||||
def _place_text(place: Any) -> str:
|
||||
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
|
||||
|
||||
|
||||
def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
|
||||
tables: Tables = TABLES) -> str:
|
||||
"""How the role is worked, one of `REMOTE` or ''.
|
||||
|
||||
`value` is the host's own (a bool, or its word; `host_word`'s 'true' and
|
||||
'false' are bools). True is remote. False says nothing more than the
|
||||
places do: SmartRecruiters, Ashby, Recruitee and Workable send it for
|
||||
every role that is not remote, hybrid ones included. A word goes
|
||||
through the host's table, else the shared map. When the value says
|
||||
nothing, the place line or the place texts are read for a word
|
||||
(`remote_of`: "Hybrid, Berlin"), and last a place the reader marked
|
||||
remote is remote."""
|
||||
if value is True or value == "true":
|
||||
return "remote"
|
||||
said = value not in (None, False, "false")
|
||||
if said and (found := lookup("remote", host, str(value), tables)):
|
||||
return found
|
||||
places = list(places)
|
||||
if found := remote_of(line or "; ".join(map(_place_text, places))):
|
||||
return found
|
||||
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
|
||||
return "remote"
|
||||
return ""
|
||||
Reference in new issue
Block a user