Files
jobboard/tests/test_prep.py
T
WolfandClaude Opus 5.5 c23594507a Score each role's closeness to the agent's CV, let the judge also read the 50 closest the score leaves out, and order CV match's unjudged roles by it
The CV is cut into chunks inside e5's window, embedded like a prompt and
cached per agent under a hash of the model, chunking and text. A role's
cv_fit (scores.cv_fit, migration 31) is its best chunk cosine, scaled with
its own calibration band: fit's clipped 88 of agent 2's ungated roles to 1.
It is no term of the Agent match score. judge_cv_top (default 50) sits
beside judge_top and judge_min on the agents page and card.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 23:35:21 +02:00

415 lines
19 KiB
Python

"""`app.prep`: the host's words mapped centrally, the stage over the due
roles, the unmapped list, the table fingerprint and migration 22. The host
tables here are fakes; each real table is held to its host by R1's and R2's
tests, and to the vocabularies by the last tests of this file."""
from __future__ import annotations
import importlib
import json
import sqlite3
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import cli, models, prep # noqa: E402
from app.prep import hosts, words # noqa: E402
from app.prep.country import country, of_host # noqa: E402
from app.storage import Store # noqa: E402
from desk.models import Item # noqa: E402
from plugins.readers import _shared # noqa: E402
TABLES: dict[str, Any] = {
"acme": {
"seniority": {"experienced": "", "sr": "senior"},
"employment": {"regular full time": "full_time", "permanent": ""},
"remote": {"2": "hybrid", "fully": "remote"},
},
"codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}},
"parts": {"match": {"mode": "substring",
"rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}},
}
# ------------------------------------------------------------------- words
@pytest.mark.parametrize(("field", "word", "value"), [
("seniority", "SR", "senior"),
("seniority", " Experienced ", ""), # known, names no level
("seniority", "senior", "senior"), # already the vocabulary
("seniority", "Grand Poobah", None), # unknown
("employment", "Regular Full Time", "full_time"),
("employment", "Permanent", ""),
("employment", "contract", "contract"),
("employment", "Zero hours", None),
("remote", "2", "hybrid"),
("remote", "On-site", "onsite"), # the shared words
("remote", "on_site", "onsite"),
("remote", "IN_OFFICE", "onsite"),
("remote", "office", "onsite"),
("remote", "Hybrid", "hybrid"),
("remote", "unspecified", None),
("seniority", "", ""),
])
def test_a_word_goes_through_the_hosts_table_then_the_vocabulary(
field: str, word: str, value: str | None) -> None:
assert words.lookup(field, "acme", word, TABLES) == value
def test_another_hosts_table_is_not_used() -> None:
assert words.lookup("seniority", "other", "sr", TABLES) is None
def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None:
assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time"
assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time"
assert words.lookup("employment", "codes", "software", TABLES) is None
assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary"
assert words.employment("parts", "permanent_full_time", TABLES) == "full_time"
def test_seniority_is_the_stated_level_and_the_titles() -> None:
assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff")
assert words.seniority("acme", "", "Engineer", TABLES) == ("", "")
def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None:
assert words.remote("acme", True, (), "", TABLES) == "remote"
assert words.remote("acme", "true", (), "", TABLES) == "remote"
assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote"
# False says nothing the places do not: hosts send it for hybrid roles too.
assert words.remote("acme", False, (), "Berlin", TABLES) == ""
assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid"
assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \
== "remote"
assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}],
"", TABLES) == "remote"
def test_a_line_in_danish_or_swedish_and_a_search_title_are_read() -> None:
assert words.remote("jobindex", None, (), "Nordhavn og mulighed for hjemmearbejde") \
== "hybrid"
assert words.remote("jobindex", None, (), "Aarhus, fuldt hjemmearbejde") == "remote"
assert words.remote("platsbanken", None, (), "Malmö, arbete på distans") == "remote"
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer - Remote") \
== "remote"
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer") == ""
def test_a_search_roles_title_counts_and_a_boards_does_not(store: Store) -> None:
store.upsert_role(item("1", title="Engineer (remote)", place="Berlin"))
store.upsert_role(item("2", title="Engineer (remote)", place="Berlin", host="jobindex",
provider="jobindex"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET source = 'search' WHERE source_id = 'jobindex:2'")
prep.run(store, tables=TABLES)
assert (row(store, "1")["remote"], row(store, "2", "jobindex")["remote"]) == ("", "remote")
# ----------------------------------------------------------------- country
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def ids(store: Store, *texts: str) -> list[int]:
gaz = store.gazetteer()
return [p for text in texts for p in gaz.resolve(text)]
def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None:
gaz = store.gazetteer()
assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK"
assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU"
assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == ""
assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB"
assert country(gaz, ids(store, "Europe")) == "EU"
assert country(gaz, [], "dk") == "DK"
assert country(gaz, ids(store, "Remote"), "Sverige") == "SE"
@pytest.mark.parametrize(("stated", "code"), [
("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"),
("", ""), ("Narnia", None)])
def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str,
code: str | None) -> None:
assert of_host(store.gazetteer(), stated) == code
# ------------------------------------------------------------------- stage
def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme",
**raw: Any) -> Item:
found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}",
title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"board": host, "company": "Acme"} | raw)
found.source = "ats"
return found
def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row:
[found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",))
return found
def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None:
store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2",
places=[{"text": "Aarhus", "remote": None}]))
raw = row(store, "1")
assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \
("SR", "Regular Full Time", "2")
assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \
("", "", "", "")
assert raw["prep_due"] == 1
assert store.role_places(raw["uid"])[0].place_id is None
done = prep.run(store, tables=TABLES)
assert (done.read, done.written, done.remapped) == (1, 1, True)
mapped = row(store, "1")
assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"],
mapped["remote"], mapped["country"], mapped["prep_due"]) == \
("senior", "senior", "full_time", "hybrid", "DK", 0)
assert store.role_places(mapped["uid"])[0].name == "Aarhus"
def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None:
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
store.upsert_role(item("2", seniority="sr"))
prep.run(store, tables=TABLES)
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
store.upsert_role(item("2", seniority="experienced"))
assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1)
# The place stays resolved when its text did not change.
assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus"
store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}]))
assert row(store, "1")["prep_due"] == 1
assert prep.run(store, tables=TABLES).read == 2
assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == ""
def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
store.upsert_role(item("1", seniority="sr"))
[read] = store.prep_rows("due")
store.upsert_role(item("1", seniority="experienced"))
store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)])
assert row(store, "1")["prep_due"] == 2
def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
store: Store) -> None:
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
prep.run(store, tables=TABLES)
assert row(store, "1")["remote"] == "hybrid"
# The host's own word wins over the line.
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="fully"))
prep.run(store, tables=TABLES)
assert row(store, "1")["remote"] == "remote"
uid = row(store, "1")["uid"]
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = 'lead', remote_model = 'onsite' "
"WHERE uid = ?", (uid,))
prep.run(store, tables=TABLES, remap=True)
mapped = row(store, "1")
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
("senior", "lead", "onsite")
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(item("1", company="ORSTED A/S"))
assert row(store, "1")["company_key"] != "ørsted"
prep.run(store, tables=TABLES)
assert row(store, "1")["company_key"] == "ørsted"
def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None:
store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified",
country="Narnia"))
store.upsert_role(item("2", seniority="Grand Poobah", remote=False))
store.upsert_role(item("3", host="other", seniority="Grand Poobah"))
done = prep.run(store, tables=TABLES)
assert done.unmapped == 4
listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()}
assert listed == {("seniority", "acme", "grand poobah"): 2,
("seniority", "other", "grand poobah"): 1,
("remote", "acme", "unspecified"): 1,
("country", "acme", "narnia"): 1}
first = {w["word"]: w["first_seen"] for w in store.unmapped()}
# A word a table learns leaves the list at the next run; the rest keep
# the moment they were first met.
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}}
prep.run(store, tables=learned)
assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \
first
assert ("seniority", "acme", "grand poobah") not in \
{(w["field"], w["host"], w["word"]) for w in store.unmapped()}
assert row(store, "1")["seniority"] == "exec"
def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None:
store.upsert_role(item("1", seniority="Wizard"))
store.upsert_role(item("2", seniority="Wizard"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'")
assert prep.run(store, tables=TABLES).remapped
assert not prep.run(store, tables=TABLES).remapped
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"wizard": "staff"}}}
done = prep.run(store, tables=learned)
assert (done.remapped, done.read) == (True, 1)
assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "")
# `--remap` maps the closed roles too.
assert prep.run(store, tables=learned, remap=True).read == 2
assert row(store, "2")["seniority"] == "staff"
def test_a_town_added_to_the_code_is_linked_at_the_next_prep(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(item("1", places=[{"text": "Ellwangen", "remote": None}]))
prep.run(store, tables=TABLES)
assert [p["text"] for p in store.unlinked_places()] == ["Ellwangen"]
assert row(store, "1")["country"] == ""
monkeypatch.setitem(models._CITIES, "ellwangen", "DE")
done = prep.run(store, tables=TABLES)
assert done.remapped
assert store.unlinked_places() == []
assert row(store, "1")["country"] == "DE"
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
def test_prep_folds_the_copies(store: Store) -> None:
for n in ("1", "2"):
store.upsert_role(item(n, place="Aarhus"))
assert prep.run(store, tables=TABLES).copies == 1
assert prep.run(store, tables=TABLES, dupes=False).copies is None
# --------------------------------------------------------------- migration
def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
path = tmp_path / "old.db"
store = Store(path)
store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True,
country="DK"))
store.upsert_role(item("2"))
store.state[prep.STATE] = "old"
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_prep_due")
for column in ("seniority_host", "employment_host", "remote_host", "country_host",
"employment", "prep_due"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
conn.execute("DROP TABLE unmapped")
conn.execute("PRAGMA user_version = 21")
migrated = Store(path)
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
assert row(migrated, "2")["remote_host"] == ""
assert prep.STATE not in migrated.state
# The first prep maps every open role from those words, the vocabulary
# values the old readers stored as themselves.
prep.run(migrated, tables={})
one = row(migrated, "1")
assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \
("senior", "full_time", "remote", "DK")
# ------------------------------------------------------------ CLI and API
def test_app_prep_runs_and_lists_the_unmapped_words(
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
path = tmp_path / "config.yaml"
path.write_text(f"database: {tmp_path / 'roledesk.db'}\n")
Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah"))
assert cli.main(["-c", str(path), "prep"]) == 0
assert json.loads(capsys.readouterr().out)["read"] == 1
assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0
assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"]
Store(tmp_path / "roledesk.db").upsert_role(item("2", place="Ellwangen"))
assert cli.main(["-c", str(path), "prep", "unlinked"]) == 0
assert capsys.readouterr().out.split() == ["1", "Ellwangen", "-:1", "Acme"]
assert cli.main(["-c", str(path), "prep", "--remap"]) == 0
assert json.loads(capsys.readouterr().out)["remapped"] is True
def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None:
from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
app.state.store.upsert_role(item("1", employment="Zero hours"))
prep.run(app.state.store, tables=TABLES)
with TestClient(app) as client:
found = client.get("/api/prep/unmapped").json()
assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \
[("employment", "acme", "zero hours", 1)]
def test_the_api_lists_the_unlinked_places(tmp_path: Path) -> None:
from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
app.state.store.upsert_role(item("1", place="Ellwangen", country="DE"))
prep.run(app.state.store, tables=TABLES)
with TestClient(app) as client:
found = client.get("/api/prep/unlinked").json()
assert found == [{"fold": "ellwangen", "text": "Ellwangen", "roles": 1,
"countries": [["DE", 1]], "companies": ["Acme"]}]
# ------------------------------------------------------------- the tables
def test_every_host_table_maps_into_its_vocabulary() -> None:
"""Passes with no table at all; R1's and R2's files are held here."""
for host, tables in hosts.load().items():
for field, table in tables.items():
if field == "match":
assert table["mode"] in hosts.MATCH_MODES, host
values = [value for _, value in table["rules"]]
field = "employment"
else:
assert all(key == key.strip().lower() for key in table), (host, field)
values = list(table.values())
assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field)
def test_a_match_table_must_say_how_it_matches(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n')
monkeypatch.setattr(hosts, "__path__", [str(tmp_path)])
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
with pytest.raises(ValueError, match=r"loose\.py has MATCH"):
hosts.load()
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n')
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
importlib.invalidate_caches()
assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]}
def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None:
assert _shared.EMPLOYMENTS == models.EMPLOYMENTS