About 190 German towns (mostly 1KOMMA5°'s) and the German states, French, Italian, Dutch, Asian and other towns, the ISO3 codes and German or French country names boards write. Towns keep their marks through spelling groups. The tests that used Paderborn, Jena and Überlingen as unknown towns use Bopfingen, Lauchheim and Ellwangen instead. Left: names that are no place, ambiguous ones (Halifax, Richmond, Victoria, Durham, Warwick) and bare codes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
413 lines
19 KiB
Python
413 lines
19 KiB
Python
"""`app.prep`: the host's words mapped centrally, the stage over the due
|
|
roles, the unmapped list, the table fingerprint and migration 22. The host
|
|
tables here are fakes; each real table is held to its host by R1's and R2's
|
|
tests, and to the vocabularies by the last tests of this file."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app import cli, models, prep # noqa: E402
|
|
from app.prep import hosts, words # noqa: E402
|
|
from app.prep.country import country, of_host # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
from desk.models import Item # noqa: E402
|
|
from plugins.readers import _shared # noqa: E402
|
|
|
|
TABLES: dict[str, Any] = {
|
|
"acme": {
|
|
"seniority": {"experienced": "", "sr": "senior"},
|
|
"employment": {"regular full time": "full_time", "permanent": ""},
|
|
"remote": {"2": "hybrid", "fully": "remote"},
|
|
},
|
|
"codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}},
|
|
"parts": {"match": {"mode": "substring",
|
|
"rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}},
|
|
}
|
|
|
|
|
|
# ------------------------------------------------------------------- words
|
|
|
|
@pytest.mark.parametrize(("field", "word", "value"), [
|
|
("seniority", "SR", "senior"),
|
|
("seniority", " Experienced ", ""), # known, names no level
|
|
("seniority", "senior", "senior"), # already the vocabulary
|
|
("seniority", "Grand Poobah", None), # unknown
|
|
("employment", "Regular Full Time", "full_time"),
|
|
("employment", "Permanent", ""),
|
|
("employment", "contract", "contract"),
|
|
("employment", "Zero hours", None),
|
|
("remote", "2", "hybrid"),
|
|
("remote", "On-site", "onsite"), # the shared words
|
|
("remote", "on_site", "onsite"),
|
|
("remote", "IN_OFFICE", "onsite"),
|
|
("remote", "office", "onsite"),
|
|
("remote", "Hybrid", "hybrid"),
|
|
("remote", "unspecified", None),
|
|
("seniority", "", ""),
|
|
])
|
|
def test_a_word_goes_through_the_hosts_table_then_the_vocabulary(
|
|
field: str, word: str, value: str | None) -> None:
|
|
assert words.lookup(field, "acme", word, TABLES) == value
|
|
|
|
|
|
def test_another_hosts_table_is_not_used() -> None:
|
|
assert words.lookup("seniority", "other", "sr", TABLES) is None
|
|
|
|
|
|
def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None:
|
|
assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time"
|
|
assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time"
|
|
assert words.lookup("employment", "codes", "software", TABLES) is None
|
|
assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary"
|
|
assert words.employment("parts", "permanent_full_time", TABLES) == "full_time"
|
|
|
|
|
|
def test_seniority_is_the_stated_level_and_the_titles() -> None:
|
|
assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff")
|
|
assert words.seniority("acme", "", "Engineer", TABLES) == ("", "")
|
|
|
|
|
|
def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None:
|
|
assert words.remote("acme", True, (), "", TABLES) == "remote"
|
|
assert words.remote("acme", "true", (), "", TABLES) == "remote"
|
|
assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote"
|
|
# False says nothing the places do not: hosts send it for hybrid roles too.
|
|
assert words.remote("acme", False, (), "Berlin", TABLES) == ""
|
|
assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid"
|
|
assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \
|
|
== "remote"
|
|
assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}],
|
|
"", TABLES) == "remote"
|
|
|
|
|
|
def test_a_line_in_danish_or_swedish_and_a_search_title_are_read() -> None:
|
|
assert words.remote("jobindex", None, (), "Nordhavn og mulighed for hjemmearbejde") \
|
|
== "hybrid"
|
|
assert words.remote("jobindex", None, (), "Aarhus, fuldt hjemmearbejde") == "remote"
|
|
assert words.remote("platsbanken", None, (), "Malmö, arbete på distans") == "remote"
|
|
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer - Remote") \
|
|
== "remote"
|
|
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer") == ""
|
|
|
|
|
|
def test_a_search_roles_title_counts_and_a_boards_does_not(store: Store) -> None:
|
|
store.upsert_role(item("1", title="Engineer (remote)", place="Berlin"))
|
|
store.upsert_role(item("2", title="Engineer (remote)", place="Berlin", host="jobindex",
|
|
provider="jobindex"))
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET source = 'search' WHERE source_id = 'jobindex:2'")
|
|
prep.run(store, tables=TABLES)
|
|
assert (row(store, "1")["remote"], row(store, "2", "jobindex")["remote"]) == ("", "remote")
|
|
|
|
|
|
# ----------------------------------------------------------------- country
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
def ids(store: Store, *texts: str) -> list[int]:
|
|
gaz = store.gazetteer()
|
|
return [p for text in texts for p in gaz.resolve(text)]
|
|
|
|
|
|
def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None:
|
|
gaz = store.gazetteer()
|
|
assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK"
|
|
assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU"
|
|
assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == ""
|
|
assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB"
|
|
assert country(gaz, ids(store, "Europe")) == "EU"
|
|
assert country(gaz, [], "dk") == "DK"
|
|
assert country(gaz, ids(store, "Remote"), "Sverige") == "SE"
|
|
|
|
|
|
@pytest.mark.parametrize(("stated", "code"), [
|
|
("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"),
|
|
("", ""), ("Narnia", None)])
|
|
def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str,
|
|
code: str | None) -> None:
|
|
assert of_host(store.gazetteer(), stated) == code
|
|
|
|
|
|
# ------------------------------------------------------------------- stage
|
|
|
|
def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme",
|
|
**raw: Any) -> Item:
|
|
found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}",
|
|
title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"board": host, "company": "Acme"} | raw)
|
|
found.source = "ats"
|
|
return found
|
|
|
|
|
|
def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row:
|
|
[found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",))
|
|
return found
|
|
|
|
|
|
def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2",
|
|
places=[{"text": "Aarhus", "remote": None}]))
|
|
raw = row(store, "1")
|
|
assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \
|
|
("SR", "Regular Full Time", "2")
|
|
assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \
|
|
("", "", "", "")
|
|
assert raw["prep_due"] == 1
|
|
assert store.role_places(raw["uid"])[0].place_id is None
|
|
|
|
done = prep.run(store, tables=TABLES)
|
|
|
|
assert (done.read, done.written, done.remapped) == (1, 1, True)
|
|
mapped = row(store, "1")
|
|
assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"],
|
|
mapped["remote"], mapped["country"], mapped["prep_due"]) == \
|
|
("senior", "senior", "full_time", "hybrid", "DK", 0)
|
|
assert store.role_places(mapped["uid"])[0].name == "Aarhus"
|
|
|
|
|
|
def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
|
|
store.upsert_role(item("2", seniority="sr"))
|
|
prep.run(store, tables=TABLES)
|
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
|
|
store.upsert_role(item("2", seniority="experienced"))
|
|
assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1)
|
|
# The place stays resolved when its text did not change.
|
|
assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus"
|
|
|
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}]))
|
|
assert row(store, "1")["prep_due"] == 1
|
|
assert prep.run(store, tables=TABLES).read == 2
|
|
assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == ""
|
|
|
|
|
|
def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority="sr"))
|
|
[read] = store.prep_rows("due")
|
|
store.upsert_role(item("1", seniority="experienced"))
|
|
store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)])
|
|
assert row(store, "1")["prep_due"] == 2
|
|
|
|
|
|
def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract(
|
|
store: Store) -> None:
|
|
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
|
|
uid = row(store, "1")["uid"]
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' "
|
|
"WHERE uid = ?", (uid,))
|
|
prep.run(store, tables=TABLES)
|
|
mapped = row(store, "1")
|
|
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
|
|
("senior", "lead", "remote")
|
|
# The host's own statement does overwrite.
|
|
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2"))
|
|
prep.run(store, tables=TABLES)
|
|
assert row(store, "1")["remote"] == "hybrid"
|
|
|
|
|
|
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
|
store.add_company("Ørsted")
|
|
store.upsert_role(item("1", company="ORSTED A/S"))
|
|
assert row(store, "1")["company_key"] != "ørsted"
|
|
prep.run(store, tables=TABLES)
|
|
assert row(store, "1")["company_key"] == "ørsted"
|
|
|
|
|
|
def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified",
|
|
country="Narnia"))
|
|
store.upsert_role(item("2", seniority="Grand Poobah", remote=False))
|
|
store.upsert_role(item("3", host="other", seniority="Grand Poobah"))
|
|
done = prep.run(store, tables=TABLES)
|
|
assert done.unmapped == 4
|
|
listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()}
|
|
assert listed == {("seniority", "acme", "grand poobah"): 2,
|
|
("seniority", "other", "grand poobah"): 1,
|
|
("remote", "acme", "unspecified"): 1,
|
|
("country", "acme", "narnia"): 1}
|
|
first = {w["word"]: w["first_seen"] for w in store.unmapped()}
|
|
|
|
# A word a table learns leaves the list at the next run; the rest keep
|
|
# the moment they were first met.
|
|
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}}
|
|
prep.run(store, tables=learned)
|
|
assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \
|
|
first
|
|
assert ("seniority", "acme", "grand poobah") not in \
|
|
{(w["field"], w["host"], w["word"]) for w in store.unmapped()}
|
|
assert row(store, "1")["seniority"] == "exec"
|
|
|
|
|
|
def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority="Wizard"))
|
|
store.upsert_role(item("2", seniority="Wizard"))
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'")
|
|
assert prep.run(store, tables=TABLES).remapped
|
|
assert not prep.run(store, tables=TABLES).remapped
|
|
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
|
|
|
|
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"wizard": "staff"}}}
|
|
done = prep.run(store, tables=learned)
|
|
assert (done.remapped, done.read) == (True, 1)
|
|
assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "")
|
|
# `--remap` maps the closed roles too.
|
|
assert prep.run(store, tables=learned, remap=True).read == 2
|
|
assert row(store, "2")["seniority"] == "staff"
|
|
|
|
|
|
def test_a_town_added_to_the_code_is_linked_at_the_next_prep(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.upsert_role(item("1", places=[{"text": "Ellwangen", "remote": None}]))
|
|
prep.run(store, tables=TABLES)
|
|
assert [p["text"] for p in store.unlinked_places()] == ["Ellwangen"]
|
|
assert row(store, "1")["country"] == ""
|
|
|
|
monkeypatch.setitem(models._CITIES, "ellwangen", "DE")
|
|
done = prep.run(store, tables=TABLES)
|
|
assert done.remapped
|
|
assert store.unlinked_places() == []
|
|
assert row(store, "1")["country"] == "DE"
|
|
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
|
|
|
|
|
|
def test_prep_folds_the_copies(store: Store) -> None:
|
|
for n in ("1", "2"):
|
|
store.upsert_role(item(n, place="Aarhus"))
|
|
assert prep.run(store, tables=TABLES).copies == 1
|
|
assert prep.run(store, tables=TABLES, dupes=False).copies is None
|
|
|
|
|
|
# --------------------------------------------------------------- migration
|
|
|
|
def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
|
path = tmp_path / "old.db"
|
|
store = Store(path)
|
|
store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True,
|
|
country="DK"))
|
|
store.upsert_role(item("2"))
|
|
store.state[prep.STATE] = "old"
|
|
with sqlite3.connect(path) as conn:
|
|
conn.execute("DROP INDEX idx_roles_prep_due")
|
|
for column in ("seniority_host", "employment_host", "remote_host", "country_host",
|
|
"employment", "prep_due"):
|
|
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
|
conn.execute("DROP TABLE unmapped")
|
|
conn.execute("PRAGMA user_version = 21")
|
|
|
|
migrated = Store(path)
|
|
|
|
assert migrated.migrations_applied == [22, 23, 24, 25]
|
|
one = row(migrated, "1")
|
|
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
|
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
|
assert row(migrated, "2")["remote_host"] == ""
|
|
assert prep.STATE not in migrated.state
|
|
# The first prep maps every open role from those words, the vocabulary
|
|
# values the old readers stored as themselves.
|
|
prep.run(migrated, tables={})
|
|
one = row(migrated, "1")
|
|
assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \
|
|
("senior", "full_time", "remote", "DK")
|
|
|
|
|
|
# ------------------------------------------------------------ CLI and API
|
|
|
|
def test_app_prep_runs_and_lists_the_unmapped_words(
|
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
|
path = tmp_path / "config.yaml"
|
|
path.write_text(f"database: {tmp_path / 'roledesk.db'}\n")
|
|
Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah"))
|
|
|
|
assert cli.main(["-c", str(path), "prep"]) == 0
|
|
assert json.loads(capsys.readouterr().out)["read"] == 1
|
|
assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0
|
|
assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"]
|
|
Store(tmp_path / "roledesk.db").upsert_role(item("2", place="Ellwangen"))
|
|
assert cli.main(["-c", str(path), "prep", "unlinked"]) == 0
|
|
assert capsys.readouterr().out.split() == ["1", "Ellwangen", "-:1", "Acme"]
|
|
assert cli.main(["-c", str(path), "prep", "--remap"]) == 0
|
|
assert json.loads(capsys.readouterr().out)["remapped"] is True
|
|
|
|
|
|
def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None:
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.config import Config, ServerConfig
|
|
from app.main import create_app
|
|
|
|
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
|
database=tmp_path / "roledesk.db"))
|
|
app.state.store.upsert_role(item("1", employment="Zero hours"))
|
|
prep.run(app.state.store, tables=TABLES)
|
|
with TestClient(app) as client:
|
|
found = client.get("/api/prep/unmapped").json()
|
|
assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \
|
|
[("employment", "acme", "zero hours", 1)]
|
|
|
|
|
|
def test_the_api_lists_the_unlinked_places(tmp_path: Path) -> None:
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.config import Config, ServerConfig
|
|
from app.main import create_app
|
|
|
|
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
|
database=tmp_path / "roledesk.db"))
|
|
app.state.store.upsert_role(item("1", place="Ellwangen", country="DE"))
|
|
prep.run(app.state.store, tables=TABLES)
|
|
with TestClient(app) as client:
|
|
found = client.get("/api/prep/unlinked").json()
|
|
assert found == [{"fold": "ellwangen", "text": "Ellwangen", "roles": 1,
|
|
"countries": [["DE", 1]], "companies": ["Acme"]}]
|
|
|
|
|
|
# ------------------------------------------------------------- the tables
|
|
|
|
def test_every_host_table_maps_into_its_vocabulary() -> None:
|
|
"""Passes with no table at all; R1's and R2's files are held here."""
|
|
for host, tables in hosts.load().items():
|
|
for field, table in tables.items():
|
|
if field == "match":
|
|
assert table["mode"] in hosts.MATCH_MODES, host
|
|
values = [value for _, value in table["rules"]]
|
|
field = "employment"
|
|
else:
|
|
assert all(key == key.strip().lower() for key in table), (host, field)
|
|
values = list(table.values())
|
|
assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field)
|
|
|
|
|
|
def test_a_match_table_must_say_how_it_matches(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n')
|
|
monkeypatch.setattr(hosts, "__path__", [str(tmp_path)])
|
|
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
|
|
with pytest.raises(ValueError, match=r"loose\.py has MATCH"):
|
|
hosts.load()
|
|
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n')
|
|
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
|
|
importlib.invalidate_caches()
|
|
assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]}
|
|
|
|
|
|
def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None:
|
|
assert _shared.EMPLOYMENTS == models.EMPLOYMENTS
|