Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
@@ -0,0 +1,110 @@
|
||||
"""A host's word to the app's vocabulary: seniority, employment and remote.
|
||||
|
||||
Each field is looked up the same way: the host's table in `hosts`, keyed by
|
||||
the word lowercased, then a word that already is a value of the vocabulary
|
||||
(what readers stored before they handed on the host's words, and what a
|
||||
host with the app's own words sends), then, for remote, the shared word map.
|
||||
An employment word may also match a host's `MATCH` fragments. A word none
|
||||
of these knows is unmapped: it maps to '' and `lookup` says so,
|
||||
so the prep stage can list it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of
|
||||
from app.prep.hosts import TABLES
|
||||
from app.seniority import from_title
|
||||
|
||||
#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`).
|
||||
Tables = Mapping[str, Mapping[str, Any]]
|
||||
|
||||
VOCABULARIES: dict[str, tuple[str, ...]] = {
|
||||
"seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE}
|
||||
|
||||
#: How hosts without a table of their own write remote, after `_squash`:
|
||||
#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office").
|
||||
REMOTE_WORDS: dict[str, str] = {
|
||||
"remote": "remote", "fullyremote": "remote", "remotefirst": "remote",
|
||||
"hybrid": "hybrid",
|
||||
"onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite",
|
||||
"officebased": "onsite", "onpremise": "onsite",
|
||||
}
|
||||
|
||||
|
||||
def _match(rules: Mapping[str, Any], key: str) -> str | None:
|
||||
"""The value of the first `MATCH` fragment `key` holds, by its mode."""
|
||||
parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set()
|
||||
for fragment, value in rules["rules"]:
|
||||
if fragment in parts if rules["mode"] == "token" else fragment in key:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def _squash(word: str) -> str:
|
||||
return "".join(c for c in word.lower() if c not in " -_")
|
||||
|
||||
|
||||
def lookup(field: str, host: str, word: str,
|
||||
tables: Tables = TABLES) -> str | None:
|
||||
"""The value `word` maps to for `host`, '' for a known word that names
|
||||
none, None for a word nothing knows. An empty word is ''."""
|
||||
key = (word or "").strip().lower()
|
||||
if not key:
|
||||
return ""
|
||||
table = tables.get(host, {})
|
||||
found = table.get(field, {}).get(key)
|
||||
if found is None and field == "employment" and "match" in table:
|
||||
found = _match(table["match"], key)
|
||||
if found is not None:
|
||||
return found
|
||||
if key in VOCABULARIES[field]:
|
||||
return key
|
||||
if field == "remote":
|
||||
return REMOTE_WORDS.get(_squash(key))
|
||||
return None
|
||||
|
||||
|
||||
def seniority(host: str, word: str, title: str,
|
||||
tables: Tables = TABLES) -> tuple[str, str]:
|
||||
"""`(stated, from the title)`: the host's word through its table, and
|
||||
`seniority.from_title`. `roles.seniority` is the model's level, else the
|
||||
stated one, else the title's (`Store.write_prep`)."""
|
||||
return lookup("seniority", host, word, tables) or "", from_title(title)
|
||||
|
||||
|
||||
def employment(host: str, word: str,
|
||||
tables: Tables = TABLES) -> str:
|
||||
return lookup("employment", host, word, tables) or ""
|
||||
|
||||
|
||||
def _place_text(place: Any) -> str:
|
||||
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
|
||||
|
||||
|
||||
def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
|
||||
tables: Tables = TABLES) -> str:
|
||||
"""How the role is worked, one of `REMOTE` or ''.
|
||||
|
||||
`value` is the host's own (a bool, or its word; `host_word`'s 'true' and
|
||||
'false' are bools). True is remote. False says nothing more than the
|
||||
places do: SmartRecruiters, Ashby, Recruitee and Workable send it for
|
||||
every role that is not remote, hybrid ones included. A word goes
|
||||
through the host's table, else the shared map. When the value says
|
||||
nothing, the place line or the place texts are read for a word
|
||||
(`remote_of`: "Hybrid, Berlin"), and last a place the reader marked
|
||||
remote is remote."""
|
||||
if value is True or value == "true":
|
||||
return "remote"
|
||||
said = value not in (None, False, "false")
|
||||
if said and (found := lookup("remote", host, str(value), tables)):
|
||||
return found
|
||||
places = list(places)
|
||||
if found := remote_of(line or "; ".join(map(_place_text, places))):
|
||||
return found
|
||||
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
|
||||
return "remote"
|
||||
return ""
|
||||
Reference in new issue
Block a user