"""`app.prep`: the host's words mapped centrally, the stage over the due roles, the unmapped list, the table fingerprint and migration 22. The host tables here are fakes; each real table is held to its host by R1's and R2's tests, and to the vocabularies by the last tests of this file.""" from __future__ import annotations import importlib import json import sqlite3 import sys from datetime import UTC, datetime from pathlib import Path from typing import Any import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app import cli, models, prep # noqa: E402 from app.prep import hosts, words # noqa: E402 from app.prep.country import country, of_host # noqa: E402 from app.storage import Store # noqa: E402 from desk.models import Item # noqa: E402 from plugins.readers import _shared # noqa: E402 TABLES: dict[str, Any] = { "acme": { "seniority": {"experienced": "", "sr": "senior"}, "employment": {"regular full time": "full_time", "permanent": ""}, "remote": {"2": "hybrid", "fully": "remote"}, }, "codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}}, "parts": {"match": {"mode": "substring", "rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}}, } # ------------------------------------------------------------------- words @pytest.mark.parametrize(("field", "word", "value"), [ ("seniority", "SR", "senior"), ("seniority", " Experienced ", ""), # known, names no level ("seniority", "senior", "senior"), # already the vocabulary ("seniority", "Grand Poobah", None), # unknown ("employment", "Regular Full Time", "full_time"), ("employment", "Permanent", ""), ("employment", "contract", "contract"), ("employment", "Zero hours", None), ("remote", "2", "hybrid"), ("remote", "On-site", "onsite"), # the shared words ("remote", "on_site", "onsite"), ("remote", "IN_OFFICE", "onsite"), ("remote", "office", "onsite"), ("remote", "Hybrid", "hybrid"), ("remote", "unspecified", None), ("seniority", "", ""), ]) def test_a_word_goes_through_the_hosts_table_then_the_vocabulary( field: str, word: str, value: str | None) -> None: assert words.lookup(field, "acme", word, TABLES) == value def test_another_hosts_table_is_not_used() -> None: assert words.lookup("seniority", "other", "sr", TABLES) is None def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None: assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time" assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time" assert words.lookup("employment", "codes", "software", TABLES) is None assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary" assert words.employment("parts", "permanent_full_time", TABLES) == "full_time" def test_seniority_is_the_stated_level_and_the_titles() -> None: assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff") assert words.seniority("acme", "", "Engineer", TABLES) == ("", "") def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None: assert words.remote("acme", True, (), "", TABLES) == "remote" assert words.remote("acme", "true", (), "", TABLES) == "remote" assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote" # False says nothing the places do not: hosts send it for hybrid roles too. assert words.remote("acme", False, (), "Berlin", TABLES) == "" assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid" assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \ == "remote" assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}], "", TABLES) == "remote" def test_a_line_in_danish_or_swedish_and_a_search_title_are_read() -> None: assert words.remote("jobindex", None, (), "Nordhavn og mulighed for hjemmearbejde") \ == "hybrid" assert words.remote("jobindex", None, (), "Aarhus, fuldt hjemmearbejde") == "remote" assert words.remote("platsbanken", None, (), "Malmö, arbete på distans") == "remote" assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer - Remote") \ == "remote" assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer") == "" def test_a_search_roles_title_counts_and_a_boards_does_not(store: Store) -> None: store.upsert_role(item("1", title="Engineer (remote)", place="Berlin")) store.upsert_role(item("2", title="Engineer (remote)", place="Berlin", host="jobindex", provider="jobindex")) with store.transaction() as conn: conn.execute("UPDATE roles SET source = 'search' WHERE source_id = 'jobindex:2'") prep.run(store, tables=TABLES) assert (row(store, "1")["remote"], row(store, "2", "jobindex")["remote"]) == ("", "remote") # ----------------------------------------------------------------- country @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") def ids(store: Store, *texts: str) -> list[int]: gaz = store.gazetteer() return [p for text in texts for p in gaz.resolve(text)] def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None: gaz = store.gazetteer() assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK" assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU" assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == "" assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB" assert country(gaz, ids(store, "Europe")) == "EU" assert country(gaz, [], "dk") == "DK" assert country(gaz, ids(store, "Remote"), "Sverige") == "SE" @pytest.mark.parametrize(("stated", "code"), [ ("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"), ("", ""), ("Narnia", None)]) def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str, code: str | None) -> None: assert of_host(store.gazetteer(), stated) == code # ------------------------------------------------------------------- stage def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme", **raw: Any) -> Item: found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}", title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"board": host, "company": "Acme"} | raw) found.source = "ats" return found def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row: [found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",)) return found def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None: store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2", places=[{"text": "Aarhus", "remote": None}])) raw = row(store, "1") assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \ ("SR", "Regular Full Time", "2") assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \ ("", "", "", "") assert raw["prep_due"] == 1 assert store.role_places(raw["uid"])[0].place_id is None done = prep.run(store, tables=TABLES) assert (done.read, done.written, done.remapped) == (1, 1, True) mapped = row(store, "1") assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"], mapped["remote"], mapped["country"], mapped["prep_due"]) == \ ("senior", "senior", "full_time", "hybrid", "DK", 0) assert store.role_places(mapped["uid"])[0].name == "Aarhus" def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None: store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}])) store.upsert_role(item("2", seniority="sr")) prep.run(store, tables=TABLES) store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}])) store.upsert_role(item("2", seniority="experienced")) assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1) # The place stays resolved when its text did not change. assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus" store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}])) assert row(store, "1")["prep_due"] == 1 assert prep.run(store, tables=TABLES).read == 2 assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == "" def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None: store.upsert_role(item("1", seniority="sr")) [read] = store.prep_rows("due") store.upsert_role(item("1", seniority="experienced")) store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)]) assert row(store, "1")["prep_due"] == 2 def test_the_model_level_and_remote_win_over_the_host_and_the_place_line( store: Store) -> None: store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin")) prep.run(store, tables=TABLES) assert row(store, "1")["remote"] == "hybrid" # The host's own word wins over the line. store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="fully")) prep.run(store, tables=TABLES) assert row(store, "1")["remote"] == "remote" uid = row(store, "1")["uid"] with store.transaction() as conn: conn.execute("UPDATE roles SET seniority_model = 'lead', remote_model = 'onsite' " "WHERE uid = ?", (uid,)) prep.run(store, tables=TABLES, remap=True) mapped = row(store, "1") assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \ ("senior", "lead", "onsite") def test_the_company_resolves_through_its_aliases(store: Store) -> None: store.add_company("Ørsted") store.upsert_role(item("1", company="ORSTED A/S")) assert row(store, "1")["company_key"] != "ørsted" prep.run(store, tables=TABLES) assert row(store, "1")["company_key"] == "ørsted" def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None: store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified", country="Narnia")) store.upsert_role(item("2", seniority="Grand Poobah", remote=False)) store.upsert_role(item("3", host="other", seniority="Grand Poobah")) done = prep.run(store, tables=TABLES) assert done.unmapped == 4 listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()} assert listed == {("seniority", "acme", "grand poobah"): 2, ("seniority", "other", "grand poobah"): 1, ("remote", "acme", "unspecified"): 1, ("country", "acme", "narnia"): 1} first = {w["word"]: w["first_seen"] for w in store.unmapped()} # A word a table learns leaves the list at the next run; the rest keep # the moment they were first met. learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}} prep.run(store, tables=learned) assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \ first assert ("seniority", "acme", "grand poobah") not in \ {(w["field"], w["host"], w["word"]) for w in store.unmapped()} assert row(store, "1")["seniority"] == "exec" def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None: store.upsert_role(item("1", seniority="Wizard")) store.upsert_role(item("2", seniority="Wizard")) with store.transaction() as conn: conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'") assert prep.run(store, tables=TABLES).remapped assert not prep.run(store, tables=TABLES).remapped assert store.state[prep.STATE] == prep.fingerprint(TABLES) learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"wizard": "staff"}}} done = prep.run(store, tables=learned) assert (done.remapped, done.read) == (True, 1) assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "") # `--remap` maps the closed roles too. assert prep.run(store, tables=learned, remap=True).read == 2 assert row(store, "2")["seniority"] == "staff" def test_a_town_added_to_the_code_is_linked_at_the_next_prep( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.upsert_role(item("1", places=[{"text": "Ellwangen", "remote": None}])) prep.run(store, tables=TABLES) assert [p["text"] for p in store.unlinked_places()] == ["Ellwangen"] assert row(store, "1")["country"] == "" monkeypatch.setitem(models._CITIES, "ellwangen", "DE") done = prep.run(store, tables=TABLES) assert done.remapped assert store.unlinked_places() == [] assert row(store, "1")["country"] == "DE" assert store.state[prep.STATE] == prep.fingerprint(TABLES) def test_prep_folds_the_copies(store: Store) -> None: for n in ("1", "2"): store.upsert_role(item(n, place="Aarhus")) assert prep.run(store, tables=TABLES).copies == 1 assert prep.run(store, tables=TABLES, dupes=False).copies is None # --------------------------------------------------------------- migration def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None: path = tmp_path / "old.db" store = Store(path) store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True, country="DK")) store.upsert_role(item("2")) store.state[prep.STATE] = "old" with sqlite3.connect(path) as conn: conn.execute("DROP INDEX idx_roles_prep_due") for column in ("seniority_host", "employment_host", "remote_host", "country_host", "employment", "prep_due"): conn.execute(f"ALTER TABLE roles DROP COLUMN {column}") conn.execute("DROP TABLE unmapped") conn.execute("PRAGMA user_version = 21") migrated = Store(path) assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28] one = row(migrated, "1") assert (one["seniority_host"], one["employment_host"], one["remote_host"], one["country_host"]) == ("senior", "full_time", "true", "DK") assert row(migrated, "2")["remote_host"] == "" assert prep.STATE not in migrated.state # The first prep maps every open role from those words, the vocabulary # values the old readers stored as themselves. prep.run(migrated, tables={}) one = row(migrated, "1") assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \ ("senior", "full_time", "remote", "DK") # ------------------------------------------------------------ CLI and API def test_app_prep_runs_and_lists_the_unmapped_words( tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: path = tmp_path / "config.yaml" path.write_text(f"database: {tmp_path / 'roledesk.db'}\n") Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah")) assert cli.main(["-c", str(path), "prep"]) == 0 assert json.loads(capsys.readouterr().out)["read"] == 1 assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0 assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"] Store(tmp_path / "roledesk.db").upsert_role(item("2", place="Ellwangen")) assert cli.main(["-c", str(path), "prep", "unlinked"]) == 0 assert capsys.readouterr().out.split() == ["1", "Ellwangen", "-:1", "Acme"] assert cli.main(["-c", str(path), "prep", "--remap"]) == 0 assert json.loads(capsys.readouterr().out)["remapped"] is True def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None: from fastapi.testclient import TestClient from app.config import Config, ServerConfig from app.main import create_app app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), database=tmp_path / "roledesk.db")) app.state.store.upsert_role(item("1", employment="Zero hours")) prep.run(app.state.store, tables=TABLES) with TestClient(app) as client: found = client.get("/api/prep/unmapped").json() assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \ [("employment", "acme", "zero hours", 1)] def test_the_api_lists_the_unlinked_places(tmp_path: Path) -> None: from fastapi.testclient import TestClient from app.config import Config, ServerConfig from app.main import create_app app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), database=tmp_path / "roledesk.db")) app.state.store.upsert_role(item("1", place="Ellwangen", country="DE")) prep.run(app.state.store, tables=TABLES) with TestClient(app) as client: found = client.get("/api/prep/unlinked").json() assert found == [{"fold": "ellwangen", "text": "Ellwangen", "roles": 1, "countries": [["DE", 1]], "companies": ["Acme"]}] # ------------------------------------------------------------- the tables def test_every_host_table_maps_into_its_vocabulary() -> None: """Passes with no table at all; R1's and R2's files are held here.""" for host, tables in hosts.load().items(): for field, table in tables.items(): if field == "match": assert table["mode"] in hosts.MATCH_MODES, host values = [value for _, value in table["rules"]] field = "employment" else: assert all(key == key.strip().lower() for key in table), (host, field) values = list(table.values()) assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field) def test_a_match_table_must_say_how_it_matches(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: (tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n') monkeypatch.setattr(hosts, "__path__", [str(tmp_path)]) monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False) with pytest.raises(ValueError, match=r"loose\.py has MATCH"): hosts.load() (tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n') monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False) importlib.invalidate_caches() assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]} def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None: assert _shared.EMPLOYMENTS == models.EMPLOYMENTS