Files
jobboard/tests/test_prep.py
T

415 lines
19 KiB
Python

"""`app.prep`: the host's words mapped centrally, the stage over the due
roles, the unmapped list, the table fingerprint and migration 22. The host
tables here are fakes; each real table is held to its host by R1's and R2's
tests, and to the vocabularies by the last tests of this file."""
from __future__ import annotations
import importlib
import json
import sqlite3
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import cli, models, prep # noqa: E402
from app.prep import hosts, words # noqa: E402
from app.prep.country import country, of_host # noqa: E402
from app.storage import Store # noqa: E402
from desk.models import Item # noqa: E402
from plugins.readers import _shared # noqa: E402
TABLES: dict[str, Any] = {
"acme": {
"seniority": {"experienced": "", "sr": "senior"},
"employment": {"regular full time": "full_time", "permanent": ""},
"remote": {"2": "hybrid", "fully": "remote"},
},
"codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}},
"parts": {"match": {"mode": "substring",
"rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}},
}
# ------------------------------------------------------------------- words
@pytest.mark.parametrize(("field", "word", "value"), [
("seniority", "SR", "senior"),
("seniority", " Experienced ", ""), # known, names no level
("seniority", "senior", "senior"), # already the vocabulary
("seniority", "Grand Poobah", None), # unknown
("employment", "Regular Full Time", "full_time"),
("employment", "Permanent", ""),
("employment", "contract", "contract"),
("employment", "Zero hours", None),
("remote", "2", "hybrid"),
("remote", "On-site", "onsite"), # the shared words
("remote", "on_site", "onsite"),
("remote", "IN_OFFICE", "onsite"),
("remote", "office", "onsite"),
("remote", "Hybrid", "hybrid"),
("remote", "unspecified", None),
("seniority", "", ""),
])
def test_a_word_goes_through_the_hosts_table_then_the_vocabulary(
field: str, word: str, value: str | None) -> None:
assert words.lookup(field, "acme", word, TABLES) == value
def test_another_hosts_table_is_not_used() -> None:
assert words.lookup("seniority", "other", "sr", TABLES) is None
def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None:
assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time"
assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time"
assert words.lookup("employment", "codes", "software", TABLES) is None
assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary"
assert words.employment("parts", "permanent_full_time", TABLES) == "full_time"
def test_seniority_is_the_stated_level_and_the_titles() -> None:
assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff")
assert words.seniority("acme", "", "Engineer", TABLES) == ("", "")
def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None:
assert words.remote("acme", True, (), "", TABLES) == "remote"
assert words.remote("acme", "true", (), "", TABLES) == "remote"
assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote"
# False says nothing the places do not: hosts send it for hybrid roles too.
assert words.remote("acme", False, (), "Berlin", TABLES) == ""
assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid"
assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \
== "remote"
assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}],
"", TABLES) == "remote"
def test_a_line_in_danish_or_swedish_and_a_search_title_are_read() -> None:
assert words.remote("jobindex", None, (), "Nordhavn og mulighed for hjemmearbejde") \
== "hybrid"
assert words.remote("jobindex", None, (), "Aarhus, fuldt hjemmearbejde") == "remote"
assert words.remote("platsbanken", None, (), "Malmö, arbete på distans") == "remote"
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer - Remote") \
== "remote"
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer") == ""
def test_a_search_roles_title_counts_and_a_boards_does_not(store: Store) -> None:
store.upsert_role(item("1", title="Engineer (remote)", place="Berlin"))
store.upsert_role(item("2", title="Engineer (remote)", place="Berlin", host="jobindex",
provider="jobindex"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET source = 'search' WHERE source_id = 'jobindex:2'")
prep.run(store, tables=TABLES)
assert (row(store, "1")["remote"], row(store, "2", "jobindex")["remote"]) == ("", "remote")
# ----------------------------------------------------------------- country
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def ids(store: Store, *texts: str) -> list[int]:
gaz = store.gazetteer()
return [p for text in texts for p in gaz.resolve(text)]
def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None:
gaz = store.gazetteer()
assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK"
assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU"
assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == ""
assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB"
assert country(gaz, ids(store, "Europe")) == "EU"
assert country(gaz, [], "dk") == "DK"
assert country(gaz, ids(store, "Remote"), "Sverige") == "SE"
@pytest.mark.parametrize(("stated", "code"), [
("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"),
("", ""), ("Narnia", None)])
def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str,
code: str | None) -> None:
assert of_host(store.gazetteer(), stated) == code
# ------------------------------------------------------------------- stage
def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme",
**raw: Any) -> Item:
found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}",
title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"board": host, "company": "Acme"} | raw)
found.source = "ats"
return found
def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row:
[found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",))
return found
def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None:
store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2",
places=[{"text": "Aarhus", "remote": None}]))
raw = row(store, "1")
assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \
("SR", "Regular Full Time", "2")
assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \
("", "", "", "")
assert raw["prep_due"] == 1
assert store.role_places(raw["uid"])[0].place_id is None
done = prep.run(store, tables=TABLES)
assert (done.read, done.written, done.remapped) == (1, 1, True)
mapped = row(store, "1")
assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"],
mapped["remote"], mapped["country"], mapped["prep_due"]) == \
("senior", "senior", "full_time", "hybrid", "DK", 0)
assert store.role_places(mapped["uid"])[0].name == "Aarhus"
def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None:
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
store.upsert_role(item("2", seniority="sr"))
prep.run(store, tables=TABLES)
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
store.upsert_role(item("2", seniority="experienced"))
assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1)
# The place stays resolved when its text did not change.
assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus"
store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}]))
assert row(store, "1")["prep_due"] == 1
assert prep.run(store, tables=TABLES).read == 2
assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == ""
def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
store.upsert_role(item("1", seniority="sr"))
[read] = store.prep_rows("due")
store.upsert_role(item("1", seniority="experienced"))
store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)])
assert row(store, "1")["prep_due"] == 2
def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
store: Store) -> None:
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
prep.run(store, tables=TABLES)
assert row(store, "1")["remote"] == "hybrid"
# The host's own word wins over the line.
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="fully"))
prep.run(store, tables=TABLES)
assert row(store, "1")["remote"] == "remote"
uid = row(store, "1")["uid"]
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = 'lead', remote_model = 'onsite' "
"WHERE uid = ?", (uid,))
prep.run(store, tables=TABLES, remap=True)
mapped = row(store, "1")
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
("senior", "lead", "onsite")
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(item("1", company="ORSTED A/S"))
assert row(store, "1")["company_key"] != "ørsted"
prep.run(store, tables=TABLES)
assert row(store, "1")["company_key"] == "ørsted"
def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None:
store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified",
country="Narnia"))
store.upsert_role(item("2", seniority="Grand Poobah", remote=False))
store.upsert_role(item("3", host="other", seniority="Grand Poobah"))
done = prep.run(store, tables=TABLES)
assert done.unmapped == 4
listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()}
assert listed == {("seniority", "acme", "grand poobah"): 2,
("seniority", "other", "grand poobah"): 1,
("remote", "acme", "unspecified"): 1,
("country", "acme", "narnia"): 1}
first = {w["word"]: w["first_seen"] for w in store.unmapped()}
# A word a table learns leaves the list at the next run; the rest keep
# the moment they were first met.
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}}
prep.run(store, tables=learned)
assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \
first
assert ("seniority", "acme", "grand poobah") not in \
{(w["field"], w["host"], w["word"]) for w in store.unmapped()}
assert row(store, "1")["seniority"] == "exec"
def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None:
store.upsert_role(item("1", seniority="Wizard"))
store.upsert_role(item("2", seniority="Wizard"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'")
assert prep.run(store, tables=TABLES).remapped
assert not prep.run(store, tables=TABLES).remapped
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"wizard": "staff"}}}
done = prep.run(store, tables=learned)
assert (done.remapped, done.read) == (True, 1)
assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "")
# `--remap` maps the closed roles too.
assert prep.run(store, tables=learned, remap=True).read == 2
assert row(store, "2")["seniority"] == "staff"
def test_a_town_added_to_the_code_is_linked_at_the_next_prep(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(item("1", places=[{"text": "Ellwangen", "remote": None}]))
prep.run(store, tables=TABLES)
assert [p["text"] for p in store.unlinked_places()] == ["Ellwangen"]
assert row(store, "1")["country"] == ""
monkeypatch.setitem(models._CITIES, "ellwangen", "DE")
done = prep.run(store, tables=TABLES)
assert done.remapped
assert store.unlinked_places() == []
assert row(store, "1")["country"] == "DE"
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
def test_prep_folds_the_copies(store: Store) -> None:
for n in ("1", "2"):
store.upsert_role(item(n, place="Aarhus"))
assert prep.run(store, tables=TABLES).copies == 1
assert prep.run(store, tables=TABLES, dupes=False).copies is None
# --------------------------------------------------------------- migration
def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
path = tmp_path / "old.db"
store = Store(path)
store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True,
country="DK"))
store.upsert_role(item("2"))
store.state[prep.STATE] = "old"
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_prep_due")
for column in ("seniority_host", "employment_host", "remote_host", "country_host",
"employment", "prep_due"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
conn.execute("DROP TABLE unmapped")
conn.execute("PRAGMA user_version = 21")
migrated = Store(path)
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
assert row(migrated, "2")["remote_host"] == ""
assert prep.STATE not in migrated.state
# The first prep maps every open role from those words, the vocabulary
# values the old readers stored as themselves.
prep.run(migrated, tables={})
one = row(migrated, "1")
assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \
("senior", "full_time", "remote", "DK")
# ------------------------------------------------------------ CLI and API
def test_app_prep_runs_and_lists_the_unmapped_words(
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
path = tmp_path / "config.yaml"
path.write_text(f"database: {tmp_path / 'roledesk.db'}\n")
Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah"))
assert cli.main(["-c", str(path), "prep"]) == 0
assert json.loads(capsys.readouterr().out)["read"] == 1
assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0
assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"]
Store(tmp_path / "roledesk.db").upsert_role(item("2", place="Ellwangen"))
assert cli.main(["-c", str(path), "prep", "unlinked"]) == 0
assert capsys.readouterr().out.split() == ["1", "Ellwangen", "-:1", "Acme"]
assert cli.main(["-c", str(path), "prep", "--remap"]) == 0
assert json.loads(capsys.readouterr().out)["remapped"] is True
def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None:
from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
app.state.store.upsert_role(item("1", employment="Zero hours"))
prep.run(app.state.store, tables=TABLES)
with TestClient(app) as client:
found = client.get("/api/prep/unmapped").json()
assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \
[("employment", "acme", "zero hours", 1)]
def test_the_api_lists_the_unlinked_places(tmp_path: Path) -> None:
from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
app.state.store.upsert_role(item("1", place="Ellwangen", country="DE"))
prep.run(app.state.store, tables=TABLES)
with TestClient(app) as client:
found = client.get("/api/prep/unlinked").json()
assert found == [{"fold": "ellwangen", "text": "Ellwangen", "roles": 1,
"countries": [["DE", 1]], "companies": ["Acme"]}]
# ------------------------------------------------------------- the tables
def test_every_host_table_maps_into_its_vocabulary() -> None:
"""Passes with no table at all; R1's and R2's files are held here."""
for host, tables in hosts.load().items():
for field, table in tables.items():
if field == "match":
assert table["mode"] in hosts.MATCH_MODES, host
values = [value for _, value in table["rules"]]
field = "employment"
else:
assert all(key == key.strip().lower() for key in table), (host, field)
values = list(table.values())
assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field)
def test_a_match_table_must_say_how_it_matches(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n')
monkeypatch.setattr(hosts, "__path__", [str(tmp_path)])
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
with pytest.raises(ValueError, match=r"loose\.py has MATCH"):
hosts.load()
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n')
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
importlib.invalidate_caches()
assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]}
def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None:
assert _shared.EMPLOYMENTS == models.EMPLOYMENTS