Files
jobboard/app/prep/stage.py
T
WolfandClaude Opus 5.5 322db49773 Title levels on the ladder: principal, senior manager, distinguished, lead split (WP A)
from_title matches a title twice, for the highest people level and the
highest individual grade. A people level wins on its own rung; a grade
above that rung lifts it one rung (Principal Engineering Manager is a
senior manager). Senior Staff is principal, Senior Principal, Distinguished
and Technical Fellow are distinguished; Associate Director, AVP and Senior
Manager are senior managers. lead is now a team lead only (Team Lead, Shift
Lead, Supervisor, Teamleiter); Tech Lead, Lead X and X Lead(er) are staff.
A slash names two levels, so Senior/Staff is staff.

- Jobylon "lead" maps to staff; Recruitee maps student_college, manager,
  senior_manager and executive.
- extract's prompt describes the ladder and its two tracks; enum from LEVELS.
- prep VERSION 2, so the next prep maps every open role again.
- Migration 25 clears seniority_model where it is staff, lead, manager or
  director, so extract reads those roles again, and sets roles.seniority to
  the first filled of model, stated and title.
- The coverage test of tests/test_seniority.py passes; its xfail is gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 13:11:26 +02:00

141 lines
5.7 KiB
Python

"""The prep stage: map what the readers handed on, then fold the copies.
A pull stores the host's own words and place texts (`Store.upsert_role`)
and marks the role due. `run` maps every due role: its company through
`company_aliases`, its places against the gazetteer, its level, contract
and remote through the host's tables (`app.prep.words`), and its country
from the places (`app.prep.country`). When the tables changed since the
last run (`fingerprint`, kept in state as `prep:tables`), every open role
is mapped again from its stored words, with no new pull. Then the words no
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
copies across the whole database.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, Sequence
from typing import Any, NamedTuple
from app import places
from app.models import company_key
from app.models import company_fold
from app.prep import words
from app.prep.country import country, of_host
from app.prep.hosts import TABLES
from app.prep.words import Tables
from app.storage import Store
#: Raised when the rules in this package change the way a stored word maps,
#: so every open role is mapped again as it is after a table changes.
VERSION = 2
#: The state key the last run's `fingerprint` is kept under.
STATE = "prep:tables"
class Prepped(NamedTuple):
"""What one run did: roles read, rows written, whether every role was
mapped again, words left unmapped, and the open copies after the fold
(None when it was not run)."""
read: int
written: int
remapped: bool
unmapped: int
copies: int | None
def fingerprint(tables: Tables = TABLES) -> str:
"""A hash of every host table, the shared remote words and `VERSION`."""
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
sort_keys=True)
return hashlib.sha1(blob.encode()).hexdigest()
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
"""The key a role from `name` belongs to, as `Store.resolve_company`
has it: the company an alias names, the name's own key first when
several share the fold, else `company_key(name)`."""
own = company_key(name)
keys = folds.get(company_fold(name), ())
return own if own in keys or not keys else keys[0]
def _places(raw: Any) -> list[Any]:
try:
found = json.loads(raw) if isinstance(raw, str) else raw
except ValueError:
return []
return found if isinstance(found, list) else []
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
host = str(row["host"] or "")
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
return {
"uid": row["uid"],
"due": row["prep_due"],
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
"seniority_stated": stated,
"seniority_title": titled,
"employment": words.employment(host, row["employment_host"], tables),
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
"remote_places": words.remote(
host, None, _places(row["raw_places"]), row["place"], tables,
title=row["title"] if row["source"] == "search" else ""),
"country": country(gaz, place_ids, row["country_host"]),
}
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
return bool(row["prep_due"]) or level != row["seniority"] \
or (found["remote"] and found["remote"] != row["remote"]) \
or (found["remote_places"] and not row["remote"]) \
or (found["country"] and found["country"] != row["country"]) \
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
"seniority_title", "employment"))
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
tables: Tables = TABLES) -> bool:
"""Whether no table, vocabulary or gazetteer knows this host word."""
if field == "country":
return of_host(gaz, word) is None
if field == "remote" and word in ("true", "false"):
return False
return words.lookup(field, host, word, tables) is None
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
dupes: bool = True) -> Prepped:
"""Map the due roles, or every open one when the tables changed, or
every role with `remap`; list the unmapped words; fold the copies."""
mark = fingerprint(tables)
whole = remap or store.state.get(STATE) != mark
rows = store.prep_rows("due")
if whole:
due = {r["uid"] for r in rows}
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
ids = store.resolve_role_places([r["uid"] for r in rows])
gaz = store.gazetteer()
folds = store.company_folds()
written = []
for row in rows:
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
if _changed(row, found):
written.append(found)
store.write_prep(written)
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
store.replace_unmapped(left)
if whole:
store.state[STATE] = mark
copies = store.rebuild_dupes().copies if dupes else None
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
copies=copies)