415 lines
19 KiB
Python
415 lines
19 KiB
Python
"""`app.prep`: the host's words mapped centrally, the stage over the due
|
|
roles, the unmapped list, the table fingerprint and migration 22. The host
|
|
tables here are fakes; each real table is held to its host by R1's and R2's
|
|
tests, and to the vocabularies by the last tests of this file."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app import cli, models, prep # noqa: E402
|
|
from app.prep import hosts, words # noqa: E402
|
|
from app.prep.country import country, of_host # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
from desk.models import Item # noqa: E402
|
|
from plugins.readers import _shared # noqa: E402
|
|
|
|
TABLES: dict[str, Any] = {
|
|
"acme": {
|
|
"seniority": {"experienced": "", "sr": "senior"},
|
|
"employment": {"regular full time": "full_time", "permanent": ""},
|
|
"remote": {"2": "hybrid", "fully": "remote"},
|
|
},
|
|
"codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}},
|
|
"parts": {"match": {"mode": "substring",
|
|
"rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}},
|
|
}
|
|
|
|
|
|
# ------------------------------------------------------------------- words
|
|
|
|
@pytest.mark.parametrize(("field", "word", "value"), [
|
|
("seniority", "SR", "senior"),
|
|
("seniority", " Experienced ", ""), # known, names no level
|
|
("seniority", "senior", "senior"), # already the vocabulary
|
|
("seniority", "Grand Poobah", None), # unknown
|
|
("employment", "Regular Full Time", "full_time"),
|
|
("employment", "Permanent", ""),
|
|
("employment", "contract", "contract"),
|
|
("employment", "Zero hours", None),
|
|
("remote", "2", "hybrid"),
|
|
("remote", "On-site", "onsite"), # the shared words
|
|
("remote", "on_site", "onsite"),
|
|
("remote", "IN_OFFICE", "onsite"),
|
|
("remote", "office", "onsite"),
|
|
("remote", "Hybrid", "hybrid"),
|
|
("remote", "unspecified", None),
|
|
("seniority", "", ""),
|
|
])
|
|
def test_a_word_goes_through_the_hosts_table_then_the_vocabulary(
|
|
field: str, word: str, value: str | None) -> None:
|
|
assert words.lookup(field, "acme", word, TABLES) == value
|
|
|
|
|
|
def test_another_hosts_table_is_not_used() -> None:
|
|
assert words.lookup("seniority", "other", "sr", TABLES) is None
|
|
|
|
|
|
def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None:
|
|
assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time"
|
|
assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time"
|
|
assert words.lookup("employment", "codes", "software", TABLES) is None
|
|
assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary"
|
|
assert words.employment("parts", "permanent_full_time", TABLES) == "full_time"
|
|
|
|
|
|
def test_seniority_is_the_stated_level_and_the_titles() -> None:
|
|
assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff")
|
|
assert words.seniority("acme", "", "Engineer", TABLES) == ("", "")
|
|
|
|
|
|
def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None:
|
|
assert words.remote("acme", True, (), "", TABLES) == "remote"
|
|
assert words.remote("acme", "true", (), "", TABLES) == "remote"
|
|
assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote"
|
|
# False says nothing the places do not: hosts send it for hybrid roles too.
|
|
assert words.remote("acme", False, (), "Berlin", TABLES) == ""
|
|
assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid"
|
|
assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \
|
|
== "remote"
|
|
assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}],
|
|
"", TABLES) == "remote"
|
|
|
|
|
|
def test_a_line_in_danish_or_swedish_and_a_search_title_are_read() -> None:
|
|
assert words.remote("jobindex", None, (), "Nordhavn og mulighed for hjemmearbejde") \
|
|
== "hybrid"
|
|
assert words.remote("jobindex", None, (), "Aarhus, fuldt hjemmearbejde") == "remote"
|
|
assert words.remote("platsbanken", None, (), "Malmö, arbete på distans") == "remote"
|
|
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer - Remote") \
|
|
== "remote"
|
|
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer") == ""
|
|
|
|
|
|
def test_a_search_roles_title_counts_and_a_boards_does_not(store: Store) -> None:
|
|
store.upsert_role(item("1", title="Engineer (remote)", place="Berlin"))
|
|
store.upsert_role(item("2", title="Engineer (remote)", place="Berlin", host="jobindex",
|
|
provider="jobindex"))
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET source = 'search' WHERE source_id = 'jobindex:2'")
|
|
prep.run(store, tables=TABLES)
|
|
assert (row(store, "1")["remote"], row(store, "2", "jobindex")["remote"]) == ("", "remote")
|
|
|
|
|
|
# ----------------------------------------------------------------- country
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
def ids(store: Store, *texts: str) -> list[int]:
|
|
gaz = store.gazetteer()
|
|
return [p for text in texts for p in gaz.resolve(text)]
|
|
|
|
|
|
def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None:
|
|
gaz = store.gazetteer()
|
|
assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK"
|
|
assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU"
|
|
assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == ""
|
|
assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB"
|
|
assert country(gaz, ids(store, "Europe")) == "EU"
|
|
assert country(gaz, [], "dk") == "DK"
|
|
assert country(gaz, ids(store, "Remote"), "Sverige") == "SE"
|
|
|
|
|
|
@pytest.mark.parametrize(("stated", "code"), [
|
|
("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"),
|
|
("", ""), ("Narnia", None)])
|
|
def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str,
|
|
code: str | None) -> None:
|
|
assert of_host(store.gazetteer(), stated) == code
|
|
|
|
|
|
# ------------------------------------------------------------------- stage
|
|
|
|
def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme",
|
|
**raw: Any) -> Item:
|
|
found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}",
|
|
title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"board": host, "company": "Acme"} | raw)
|
|
found.source = "ats"
|
|
return found
|
|
|
|
|
|
def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row:
|
|
[found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",))
|
|
return found
|
|
|
|
|
|
def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2",
|
|
places=[{"text": "Aarhus", "remote": None}]))
|
|
raw = row(store, "1")
|
|
assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \
|
|
("SR", "Regular Full Time", "2")
|
|
assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \
|
|
("", "", "", "")
|
|
assert raw["prep_due"] == 1
|
|
assert store.role_places(raw["uid"])[0].place_id is None
|
|
|
|
done = prep.run(store, tables=TABLES)
|
|
|
|
assert (done.read, done.written, done.remapped) == (1, 1, True)
|
|
mapped = row(store, "1")
|
|
assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"],
|
|
mapped["remote"], mapped["country"], mapped["prep_due"]) == \
|
|
("senior", "senior", "full_time", "hybrid", "DK", 0)
|
|
assert store.role_places(mapped["uid"])[0].name == "Aarhus"
|
|
|
|
|
|
def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
|
|
store.upsert_role(item("2", seniority="sr"))
|
|
prep.run(store, tables=TABLES)
|
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
|
|
store.upsert_role(item("2", seniority="experienced"))
|
|
assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1)
|
|
# The place stays resolved when its text did not change.
|
|
assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus"
|
|
|
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}]))
|
|
assert row(store, "1")["prep_due"] == 1
|
|
assert prep.run(store, tables=TABLES).read == 2
|
|
assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == ""
|
|
|
|
|
|
def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority="sr"))
|
|
[read] = store.prep_rows("due")
|
|
store.upsert_role(item("1", seniority="experienced"))
|
|
store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)])
|
|
assert row(store, "1")["prep_due"] == 2
|
|
|
|
|
|
def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
|
|
store: Store) -> None:
|
|
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
|
|
prep.run(store, tables=TABLES)
|
|
assert row(store, "1")["remote"] == "hybrid"
|
|
# The host's own word wins over the line.
|
|
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="fully"))
|
|
prep.run(store, tables=TABLES)
|
|
assert row(store, "1")["remote"] == "remote"
|
|
uid = row(store, "1")["uid"]
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET seniority_model = 'lead', remote_model = 'onsite' "
|
|
"WHERE uid = ?", (uid,))
|
|
prep.run(store, tables=TABLES, remap=True)
|
|
mapped = row(store, "1")
|
|
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
|
|
("senior", "lead", "onsite")
|
|
|
|
|
|
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
|
store.add_company("Ørsted")
|
|
store.upsert_role(item("1", company="ORSTED A/S"))
|
|
assert row(store, "1")["company_key"] != "ørsted"
|
|
prep.run(store, tables=TABLES)
|
|
assert row(store, "1")["company_key"] == "ørsted"
|
|
|
|
|
|
def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified",
|
|
country="Narnia"))
|
|
store.upsert_role(item("2", seniority="Grand Poobah", remote=False))
|
|
store.upsert_role(item("3", host="other", seniority="Grand Poobah"))
|
|
done = prep.run(store, tables=TABLES)
|
|
assert done.unmapped == 4
|
|
listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()}
|
|
assert listed == {("seniority", "acme", "grand poobah"): 2,
|
|
("seniority", "other", "grand poobah"): 1,
|
|
("remote", "acme", "unspecified"): 1,
|
|
("country", "acme", "narnia"): 1}
|
|
first = {w["word"]: w["first_seen"] for w in store.unmapped()}
|
|
|
|
# A word a table learns leaves the list at the next run; the rest keep
|
|
# the moment they were first met.
|
|
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}}
|
|
prep.run(store, tables=learned)
|
|
assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \
|
|
first
|
|
assert ("seniority", "acme", "grand poobah") not in \
|
|
{(w["field"], w["host"], w["word"]) for w in store.unmapped()}
|
|
assert row(store, "1")["seniority"] == "exec"
|
|
|
|
|
|
def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None:
|
|
store.upsert_role(item("1", seniority="Wizard"))
|
|
store.upsert_role(item("2", seniority="Wizard"))
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'")
|
|
assert prep.run(store, tables=TABLES).remapped
|
|
assert not prep.run(store, tables=TABLES).remapped
|
|
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
|
|
|
|
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"wizard": "staff"}}}
|
|
done = prep.run(store, tables=learned)
|
|
assert (done.remapped, done.read) == (True, 1)
|
|
assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "")
|
|
# `--remap` maps the closed roles too.
|
|
assert prep.run(store, tables=learned, remap=True).read == 2
|
|
assert row(store, "2")["seniority"] == "staff"
|
|
|
|
|
|
def test_a_town_added_to_the_code_is_linked_at_the_next_prep(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.upsert_role(item("1", places=[{"text": "Ellwangen", "remote": None}]))
|
|
prep.run(store, tables=TABLES)
|
|
assert [p["text"] for p in store.unlinked_places()] == ["Ellwangen"]
|
|
assert row(store, "1")["country"] == ""
|
|
|
|
monkeypatch.setitem(models._CITIES, "ellwangen", "DE")
|
|
done = prep.run(store, tables=TABLES)
|
|
assert done.remapped
|
|
assert store.unlinked_places() == []
|
|
assert row(store, "1")["country"] == "DE"
|
|
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
|
|
|
|
|
|
def test_prep_folds_the_copies(store: Store) -> None:
|
|
for n in ("1", "2"):
|
|
store.upsert_role(item(n, place="Aarhus"))
|
|
assert prep.run(store, tables=TABLES).copies == 1
|
|
assert prep.run(store, tables=TABLES, dupes=False).copies is None
|
|
|
|
|
|
# --------------------------------------------------------------- migration
|
|
|
|
def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
|
path = tmp_path / "old.db"
|
|
store = Store(path)
|
|
store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True,
|
|
country="DK"))
|
|
store.upsert_role(item("2"))
|
|
store.state[prep.STATE] = "old"
|
|
with sqlite3.connect(path) as conn:
|
|
conn.execute("DROP INDEX idx_roles_prep_due")
|
|
for column in ("seniority_host", "employment_host", "remote_host", "country_host",
|
|
"employment", "prep_due"):
|
|
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
|
conn.execute("DROP TABLE unmapped")
|
|
conn.execute("PRAGMA user_version = 21")
|
|
|
|
migrated = Store(path)
|
|
|
|
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
|
one = row(migrated, "1")
|
|
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
|
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
|
assert row(migrated, "2")["remote_host"] == ""
|
|
assert prep.STATE not in migrated.state
|
|
# The first prep maps every open role from those words, the vocabulary
|
|
# values the old readers stored as themselves.
|
|
prep.run(migrated, tables={})
|
|
one = row(migrated, "1")
|
|
assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \
|
|
("senior", "full_time", "remote", "DK")
|
|
|
|
|
|
# ------------------------------------------------------------ CLI and API
|
|
|
|
def test_app_prep_runs_and_lists_the_unmapped_words(
|
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
|
path = tmp_path / "config.yaml"
|
|
path.write_text(f"database: {tmp_path / 'roledesk.db'}\n")
|
|
Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah"))
|
|
|
|
assert cli.main(["-c", str(path), "prep"]) == 0
|
|
assert json.loads(capsys.readouterr().out)["read"] == 1
|
|
assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0
|
|
assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"]
|
|
Store(tmp_path / "roledesk.db").upsert_role(item("2", place="Ellwangen"))
|
|
assert cli.main(["-c", str(path), "prep", "unlinked"]) == 0
|
|
assert capsys.readouterr().out.split() == ["1", "Ellwangen", "-:1", "Acme"]
|
|
assert cli.main(["-c", str(path), "prep", "--remap"]) == 0
|
|
assert json.loads(capsys.readouterr().out)["remapped"] is True
|
|
|
|
|
|
def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None:
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.config import Config, ServerConfig
|
|
from app.main import create_app
|
|
|
|
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
|
database=tmp_path / "roledesk.db"))
|
|
app.state.store.upsert_role(item("1", employment="Zero hours"))
|
|
prep.run(app.state.store, tables=TABLES)
|
|
with TestClient(app) as client:
|
|
found = client.get("/api/prep/unmapped").json()
|
|
assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \
|
|
[("employment", "acme", "zero hours", 1)]
|
|
|
|
|
|
def test_the_api_lists_the_unlinked_places(tmp_path: Path) -> None:
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.config import Config, ServerConfig
|
|
from app.main import create_app
|
|
|
|
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
|
database=tmp_path / "roledesk.db"))
|
|
app.state.store.upsert_role(item("1", place="Ellwangen", country="DE"))
|
|
prep.run(app.state.store, tables=TABLES)
|
|
with TestClient(app) as client:
|
|
found = client.get("/api/prep/unlinked").json()
|
|
assert found == [{"fold": "ellwangen", "text": "Ellwangen", "roles": 1,
|
|
"countries": [["DE", 1]], "companies": ["Acme"]}]
|
|
|
|
|
|
# ------------------------------------------------------------- the tables
|
|
|
|
def test_every_host_table_maps_into_its_vocabulary() -> None:
|
|
"""Passes with no table at all; R1's and R2's files are held here."""
|
|
for host, tables in hosts.load().items():
|
|
for field, table in tables.items():
|
|
if field == "match":
|
|
assert table["mode"] in hosts.MATCH_MODES, host
|
|
values = [value for _, value in table["rules"]]
|
|
field = "employment"
|
|
else:
|
|
assert all(key == key.strip().lower() for key in table), (host, field)
|
|
values = list(table.values())
|
|
assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field)
|
|
|
|
|
|
def test_a_match_table_must_say_how_it_matches(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n')
|
|
monkeypatch.setattr(hosts, "__path__", [str(tmp_path)])
|
|
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
|
|
with pytest.raises(ValueError, match=r"loose\.py has MATCH"):
|
|
hosts.load()
|
|
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n')
|
|
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
|
|
importlib.invalidate_caches()
|
|
assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]}
|
|
|
|
|
|
def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None:
|
|
assert _shared.EMPLOYMENTS == models.EMPLOYMENTS
|