Places as a gazetteer, role_places, teaser and three seniority sources
Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
bbd7522163
10 files changed
+1437
-98
No files matched your search
@@ -164,13 +164,17 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
||||
store.upsert_role(item)
|
||||
store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))])
|
||||
with sqlite3.connect(path) as conn:
|
||||
for table in ("role_places", "place_includes", "place_aliases", "places"):
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
conn.execute("DROP INDEX idx_roles_dup_of")
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
|
||||
conn.execute("PRAGMA user_version = 17")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [18]
|
||||
assert migrated.migrations_applied == [18, 19]
|
||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
"""The gazetteer: its seed, the resolver, and a role's places in the store.
|
||||
|
||||
The store tests go through `Store`, as the feed and the gates do. The
|
||||
real-line test compares the join over the gazetteer with the whole-word rule
|
||||
it replaced (`places.matches`) on the live lines `test_places` holds: the
|
||||
join must find every role the word rule found, but for the lines listed where
|
||||
the word rule was wrong.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from desk.models import Item
|
||||
from app import places
|
||||
from app.models import Gate, _CITIES, _COUNTRY_NAMES
|
||||
from app.places import Gazetteer, fold, seed
|
||||
from app.storage import Store
|
||||
|
||||
from test_places import LINES, ORACLE_TOKENS
|
||||
|
||||
|
||||
def gazetteer() -> Gazetteer:
|
||||
seeds, includes = seed()
|
||||
ids: dict[tuple[str, str], int] = {}
|
||||
rows, aliases = [], []
|
||||
for n, s in enumerate(seeds, 1):
|
||||
ids[(s.kind, s.key)] = n
|
||||
rows.append((n, s.kind, s.key, s.name, ids[s.parent] if s.parent else None))
|
||||
aliases += [(a, n) for a in s.aliases]
|
||||
return Gazetteer(rows, aliases, [(ids[a], ids[b]) for a, b in includes])
|
||||
|
||||
|
||||
GAZ = gazetteer()
|
||||
|
||||
|
||||
def names(text: str) -> list[str]:
|
||||
return [GAZ.places[p].name for p in GAZ.resolve(text)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- seed
|
||||
|
||||
def test_the_seed_holds_the_dictionaries_in_code() -> None:
|
||||
seeds, includes = seed()
|
||||
kinds = Counter(s.kind for s in seeds)
|
||||
assert kinds["continent"] == 1
|
||||
assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line)
|
||||
keys = {(s.kind, s.key) for s in seeds}
|
||||
assert {("region", key) for key in places.REGIONS} <= keys
|
||||
assert {("region", "oresund"), ("region", "skane")} in [set(i) for i in includes]
|
||||
folds = {a for s in seeds for a in s.aliases}
|
||||
assert set(_CITIES) - {"region hovedstaden"} <= folds
|
||||
assert {"kobenhavn", "kopenhamn", "copenhagen", "skane", "emea", "eu"} <= folds
|
||||
|
||||
|
||||
def test_parents_come_before_their_children() -> None:
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for s in seed()[0]:
|
||||
assert s.parent is None or s.parent in seen, s
|
||||
seen.add((s.kind, s.key))
|
||||
|
||||
|
||||
def test_copenhagen_is_in_the_capital_region_in_denmark_in_europe() -> None:
|
||||
copenhagen = GAZ.by_key[("city", "copenhagen")]
|
||||
above = {GAZ.places[p].key for p in GAZ.ancestors(copenhagen)}
|
||||
assert above == {"hovedstaden", "oresund", "DK", "europe"}
|
||||
assert GAZ.country(copenhagen) == "DK"
|
||||
assert GAZ.by_key[("country", "US")] not in GAZ.under({GAZ.by_key[("continent", "europe")]})
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ resolving
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
("Aarhus N, Region Central Jutland, Denmark", ["Aarhus"]),
|
||||
("København K", ["Copenhagen"]),
|
||||
("Copenhagen Office", ["Copenhagen"]),
|
||||
("Malmö/Lund", ["Malmö", "Lund"]),
|
||||
("Aalborg, Copenhagen, Kolding", ["Aalborg", "Copenhagen", "Kolding"]),
|
||||
("Paderborn, Germany", ["Germany"]),
|
||||
("Skåne län, Sweden", ["Skåne"]),
|
||||
("London, Ontario", ["Ontario"]),
|
||||
("Bergen op Zoom, Netherlands", ["Netherlands"]),
|
||||
("Mexicali, BAJA CALIFORNIA, Mexico", ["Mexico"]),
|
||||
("Paris, TX", ["United States"]),
|
||||
("Remote (DE)", ["Germany"]),
|
||||
("London OR Paris OR Germany", ["London", "Paris", "Germany"]),
|
||||
("Allerød or Stockholm (SE)", ["Allerød", "Stockholm"]),
|
||||
("Finland, København, Litauen, Polen, Sverige", ["Finland", "Copenhagen", "Sweden"]),
|
||||
("Remote, EU or US", ["Europe", "United States"]),
|
||||
("Home based - EMEA", ["Europe"]),
|
||||
("Luxembourg", ["Luxembourg"]),
|
||||
("Nordhavn og mulighed for hjemmearbejde", ["Nordhavn"]),
|
||||
("Remote", []),
|
||||
("Oberkochen", []),
|
||||
("", []),
|
||||
])
|
||||
def test_a_text_resolves_to_the_places_it_names(text: str, expected: list[str]) -> None:
|
||||
assert names(text) == expected
|
||||
|
||||
|
||||
def test_luxembourg_the_town_is_in_luxembourg_the_country() -> None:
|
||||
town = GAZ.by_key[("city", "luxembourg")]
|
||||
assert GAZ.resolve("Luxembourg") == (town,)
|
||||
assert GAZ.country(town) == "LU"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["Remote", "Hybrid", "Multiple Locations", "2 Locations",
|
||||
"Remote job", "", " "])
|
||||
def test_a_text_of_how_a_role_is_worked_is_no_place(text: str) -> None:
|
||||
assert places.placeless(text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Jena"])
|
||||
def test_a_text_with_a_name_is_a_place(text: str) -> None:
|
||||
assert not places.placeless(text)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- tokens
|
||||
|
||||
def test_a_token_names_places_before_what_lies_in_them() -> None:
|
||||
assert GAZ.lookup("country:DK") == {GAZ.by_key[("country", "DK")]}
|
||||
assert GAZ.lookup("country:EU") == {GAZ.by_key[("continent", "europe")]}
|
||||
assert GAZ.lookup("region:Skåne") == {GAZ.by_key[("region", "skane")]}
|
||||
assert GAZ.lookup("place:kobenhavn") == {GAZ.by_key[("city", "copenhagen")]}
|
||||
assert GAZ.lookup("place:denmark") == {GAZ.by_key[("country", "DK")]}
|
||||
assert GAZ.lookup("place:oberkochen") is None
|
||||
assert GAZ.lookup("country:ZZ") is None
|
||||
|
||||
|
||||
def test_europe_is_every_european_country_and_eu() -> None:
|
||||
under = GAZ.under(GAZ.lookup("country:EU") or ())
|
||||
codes = GAZ.codes(under)
|
||||
assert {"DK", "SE", "DE", "GB", "CH", "EU"} <= codes
|
||||
assert not {"US", "CN"} & codes
|
||||
|
||||
|
||||
def test_oresund_holds_both_capital_regions() -> None:
|
||||
under = GAZ.under(GAZ.lookup("region:oresund") or ())
|
||||
assert {GAZ.by_key[("city", "malmo")], GAZ.by_key[("city", "copenhagen")]} <= under
|
||||
assert GAZ.by_key[("city", "aarhus")] not in under
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- store
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Iterator[Store]:
|
||||
yield Store(tmp_path / "roledesk.db")
|
||||
|
||||
|
||||
def posting(source_id: str, *, title: str = "Backend Engineer", body: str = "",
|
||||
**raw: object) -> Item:
|
||||
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
||||
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw=dict(raw))
|
||||
item.source = "ats"
|
||||
return item
|
||||
|
||||
|
||||
def texts(store: Store, uid: str) -> list[tuple[int, str, str]]:
|
||||
return [(p.position, p.text, p.name) for p in store.role_places(uid)]
|
||||
|
||||
|
||||
def test_a_fresh_file_is_seeded(store: Store) -> None:
|
||||
gaz = store.gazetteer()
|
||||
assert len(gaz.places) == len(GAZ.places)
|
||||
assert ("city", "copenhagen") in gaz.by_key
|
||||
|
||||
|
||||
def test_a_joined_line_is_several_places(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
|
||||
uid = posting("1").uid
|
||||
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
|
||||
(2, "Warsaw", "Warsaw")]
|
||||
role = store.role(uid)
|
||||
assert role is not None and role.place == "Copenhagen; London; Warsaw"
|
||||
assert [p.name for p in role.places] == ["Copenhagen", "London", "Warsaw"]
|
||||
|
||||
|
||||
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Berlin, Munich", places=[
|
||||
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
|
||||
{"text": "Remote (DE)", "remote": True}]))
|
||||
assert texts(store, posting("1").uid) == [
|
||||
(0, "Berlin", "Berlin"), (1, "Munich", "Munich"), (2, "Remote (DE)", "Germany")]
|
||||
|
||||
|
||||
def test_places_without_a_line_make_the_line(store: Store) -> None:
|
||||
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
|
||||
{"text": "Malmö", "remote": None}]))
|
||||
role = store.role(posting("1").uid)
|
||||
assert role is not None
|
||||
assert (role.place, role.country) == ("Lund; Malmö", "SE")
|
||||
|
||||
|
||||
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Aarhus"))
|
||||
store.upsert_role(posting("1"))
|
||||
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
|
||||
store.upsert_role(posting("1", place="Odense"))
|
||||
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
|
||||
|
||||
|
||||
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Aarhus"))
|
||||
uid = posting("1").uid
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
||||
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
|
||||
store.upsert_role(posting("1", place="Aalborg"))
|
||||
assert [(p.text, p.url) for p in store.role_places(uid)] == [
|
||||
("Aalborg", ""), ("Odense", "https://copy")]
|
||||
|
||||
|
||||
def test_an_extracted_place_is_resolved(store: Store) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
uid = posting("1").uid
|
||||
store.set_extracted(uid, {"place": "Lund, Sweden"})
|
||||
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
|
||||
|
||||
|
||||
# ---------------------------------------------------------- filter and gates
|
||||
|
||||
def uids(store: Store, *tokens: str) -> set[str]:
|
||||
return {r.uid for r in store.roles(limit=100_000, places=list(tokens), by_score=False)}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def several(store: Store) -> dict[str, str]:
|
||||
lines = {"cph": "Copenhagen; London", "eu": "Berlin, Germany; Paris, France",
|
||||
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
|
||||
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
|
||||
for key, line in lines.items():
|
||||
store.upsert_role(posting(key, place=line))
|
||||
return {key: posting(key).uid for key in lines}
|
||||
|
||||
|
||||
def test_a_country_is_every_place_in_it(store: Store, several: dict[str, str]) -> None:
|
||||
assert uids(store, "country:DK") == {several["cph"], several["billund"]}
|
||||
assert uids(store, "country:SE") == {several["malmo"], several["lund"], several["remote"]}
|
||||
assert uids(store, "country:DE") == {several["eu"]}
|
||||
assert uids(store, "country:GB") == {several["cph"]}
|
||||
|
||||
|
||||
def test_europe_is_every_european_role(store: Store, several: dict[str, str]) -> None:
|
||||
assert uids(store, "country:EU") == set(several.values()) - {several["ny"], several["zeiss"]}
|
||||
|
||||
|
||||
def test_a_region_is_every_town_in_it(store: Store, several: dict[str, str]) -> None:
|
||||
assert uids(store, "region:skane") == {several["malmo"], several["lund"]}
|
||||
assert uids(store, "region:oresund") == {several["malmo"], several["lund"], several["cph"]}
|
||||
assert uids(store, "place:skane") == uids(store, "region:skane")
|
||||
|
||||
|
||||
def test_a_town_is_its_place_and_lund_is_not_in_billund(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
assert uids(store, "place:lund") == {several["lund"]}
|
||||
assert uids(store, "place:kobenhavn") == {several["cph"]}
|
||||
|
||||
|
||||
def test_a_town_the_gazetteer_lacks_is_found_by_its_words(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
assert uids(store, "place:oberkochen") == {several["zeiss"]}
|
||||
|
||||
|
||||
def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
store.roles(places=["region:atlantis"])
|
||||
|
||||
|
||||
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
|
||||
store.upsert_role(posting("1", country="DK"))
|
||||
assert uids(store, "country:DK") == {posting("1").uid}
|
||||
|
||||
|
||||
def gates(store: Store, gate: Gate) -> dict[str, int]:
|
||||
agent = store.save_agent("A", gates=[gate])
|
||||
return {c.uid: c.gates[0] for c in store.score_candidates(agent.id, agent.gates)}
|
||||
|
||||
|
||||
def test_a_country_gate_is_met_by_any_place_in_the_country(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
found = gates(store, Gate(field="country", values=["DK", "SE"], mode="any"))
|
||||
assert {uid for uid, g in found.items() if g == 1} == {
|
||||
several[k] for k in ("cph", "billund", "malmo", "lund", "remote")}
|
||||
# Only a role with neither a country nor a known place has no evidence.
|
||||
assert {uid for uid, g in found.items() if g == -1} == {several["zeiss"]}
|
||||
assert found[several["ny"]] == 0 and found[several["eu"]] == 0
|
||||
|
||||
|
||||
def test_the_eu_value_of_a_country_gate_is_europe(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
found = gates(store, Gate(field="country", values=["EU"]))
|
||||
assert found[several["eu"]] == 1 and found[several["cph"]] == 1
|
||||
assert found[several["ny"]] == 0
|
||||
|
||||
|
||||
def test_a_place_gate_matches_over_the_gazetteer(store: Store,
|
||||
several: dict[str, str]) -> None:
|
||||
found = gates(store, Gate(field="place", values=["Malmö", "Lund", "oberkochen"],
|
||||
mode="any"))
|
||||
assert {uid for uid, g in found.items() if g == 1} == {
|
||||
several["malmo"], several["lund"], several["zeiss"]}
|
||||
|
||||
|
||||
# ------------------------------------------------------------ unresolved texts
|
||||
|
||||
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
|
||||
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
|
||||
store.upsert_role(posting(str(n), place=line))
|
||||
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
|
||||
("jena", "Jena", 1)]
|
||||
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
|
||||
|
||||
|
||||
def test_an_alias_links_what_it_names(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Kbh. K"))
|
||||
store.upsert_role(posting("2", place="Copenhagen"))
|
||||
uid = posting("1").uid
|
||||
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
|
||||
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
|
||||
assert texts(store, uid) == [(0, "Kbh. K", "Copenhagen")]
|
||||
assert store.unresolved_places() == []
|
||||
assert uids(store, "country:DK") == {uid, posting("2").uid}
|
||||
# The second store, another process, sees the new alias too.
|
||||
assert Store(store.path).gazetteer().lookup("place:kbh") == \
|
||||
store.gazetteer().lookup("place:copenhagen")
|
||||
|
||||
|
||||
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Oberkochen; Jena"))
|
||||
store.upsert_role(posting("2", place="Paderborn, Germany"))
|
||||
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
|
||||
store.add_place("Paderborn", "city", "country:DE")
|
||||
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
|
||||
assert texts(store, posting("2").uid) == [(0, "Paderborn, Germany", "Paderborn")]
|
||||
assert uids(store, "place:oberkochen") == {posting("1").uid}
|
||||
assert store.gazetteer().country(oberkochen) == "DE"
|
||||
with pytest.raises(ValueError):
|
||||
store.add_place("Oberkochen", "city", "country:DE")
|
||||
with pytest.raises(ValueError):
|
||||
store.add_place("Atlantis", "country", "country:EU")
|
||||
|
||||
|
||||
def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
store.add_place_alias("Kbh", "place:atlantis")
|
||||
with pytest.raises(ValueError):
|
||||
store.add_place_alias("!!", "place:copenhagen")
|
||||
|
||||
|
||||
def test_relink_places_resolves_every_role_again(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Oberkochen"))
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
conn.execute("UPDATE role_places SET place_id = "
|
||||
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
|
||||
assert store.relink_places() == 1
|
||||
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "")]
|
||||
|
||||
|
||||
# --------------------------------------------------------- against the words
|
||||
|
||||
#: Lines of `test_places.LINES` the word rule found for a token and the
|
||||
#: gazetteer does not, each rightly: the London in Canada, and a town named
|
||||
#: after Zurich in Illinois.
|
||||
NOT_FOUND_ANY_MORE: dict[str, set[str]] = {
|
||||
"country:GB": {"London, Canada; Remote, United States; Oak Brook, United States"},
|
||||
"place:london": {"London, Canada; Remote, United States; Oak Brook, United States"},
|
||||
"place:zurich": {"Lake Zurich, Illinois, United States"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tuple[str, str]]]:
|
||||
store = Store(tmp_path_factory.mktemp("real") / "roledesk.db")
|
||||
lines = {}
|
||||
for n, (place, country) in enumerate(LINES):
|
||||
item = posting(f"line{n}", place=place, country=country)
|
||||
store.upsert_role(item)
|
||||
lines[item.uid] = (place, country)
|
||||
return store, lines
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token", ORACLE_TOKENS)
|
||||
def test_the_gazetteer_finds_what_the_words_found(
|
||||
real: tuple[Store, dict[str, tuple[str, str]]], token: str) -> None:
|
||||
store, lines = real
|
||||
by_words = {uid for uid, (place, country) in lines.items()
|
||||
if places.matches(fold(place), country, token)}
|
||||
by_places = uids(store, token)
|
||||
lost = {lines[uid][0] for uid in by_words - by_places}
|
||||
assert lost <= NOT_FOUND_ANY_MORE.get(token, set()), (token, lost)
|
||||
+139
-12
@@ -14,8 +14,8 @@ from desk.models import Item
|
||||
import numpy as np
|
||||
|
||||
from app.models import (
|
||||
LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key,
|
||||
posted_at, posted_host, title_key,
|
||||
LEVELS, Application, Gate, Limits, Prompt, company_key, place_key,
|
||||
posted_at, posted_host, title_key, trusted_posted,
|
||||
)
|
||||
from app.storage.store import pack
|
||||
from app.storage import LATEST, Store
|
||||
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 18
|
||||
assert LATEST == 19
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -88,8 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
|
||||
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
|
||||
|
||||
|
||||
def _undo_v19(conn: sqlite3.Connection) -> None:
|
||||
"""A fresh file put back to the v18 shape: no gazetteer, no role places,
|
||||
no teaser, one level."""
|
||||
for table in ("role_places", "place_includes", "place_aliases", "places"):
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
|
||||
|
||||
def _undo_v18(conn: sqlite3.Connection) -> None:
|
||||
"""A fresh file put back to the v17 shape: no copies."""
|
||||
_undo_v19(conn)
|
||||
conn.execute("DROP INDEX idx_roles_dup_of")
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
|
||||
|
||||
@@ -166,7 +176,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18]
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||
@@ -371,7 +381,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18]
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19]
|
||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||
Prompt(name="Backend", text="python backend")]
|
||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||
@@ -395,7 +405,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [16, 17, 18]
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||
@@ -424,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [17, 18]
|
||||
assert migrated.migrations_applied == [17, 18, 19]
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||
"SELECT uid, place_fold, seniority FROM roles")}
|
||||
@@ -517,7 +527,7 @@ def test_posted_host_reads_board_then_provider() -> None:
|
||||
|
||||
|
||||
def test_posted_at_trusts_only_a_known_host() -> None:
|
||||
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS
|
||||
assert trusted_posted("greenhouse") and not trusted_posted("taleo")
|
||||
early = "2020-01-01T00:00:00+00:00"
|
||||
late = "2026-09-23T09:00:00+00:00"
|
||||
# A trusted host's earlier date wins over first_seen...
|
||||
@@ -527,6 +537,29 @@ def test_posted_at_trusts_only_a_known_host() -> None:
|
||||
assert posted_at(early, late, "") == late
|
||||
|
||||
|
||||
def test_a_reader_that_declares_reads_decides_its_own_date_trust(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`fields.posted` of a declared reader wins over the fallback set, either
|
||||
way; a reader without `READS`, or a provider that is none, keeps it."""
|
||||
import app.models as models
|
||||
|
||||
monkeypatch.setattr(models, "_declared_posted",
|
||||
lambda: {"greenhouse": "listed", "gem": "trusted"})
|
||||
assert not trusted_posted("greenhouse")
|
||||
assert trusted_posted("gem")
|
||||
assert trusted_posted("jobindex")
|
||||
assert not trusted_posted("taleo")
|
||||
|
||||
|
||||
def test_the_declarations_are_read_off_the_readers() -> None:
|
||||
import app.models as models
|
||||
from plugins.readers import READERS
|
||||
|
||||
declared = models._declared_posted()
|
||||
assert set(declared) <= set(READERS)
|
||||
assert all(value in ("trusted", "listed", "none") for value in declared.values())
|
||||
|
||||
|
||||
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
|
||||
first_seen = "2026-09-23T09:00:00+00:00"
|
||||
future = "2026-09-24T00:00:00+00:00" # after first_seen
|
||||
@@ -574,6 +607,92 @@ def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store)
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
|
||||
|
||||
def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
|
||||
store: Store) -> None:
|
||||
from app.jobs.embed import document
|
||||
|
||||
listed = posting(teaser="Build the payments core.")
|
||||
listed.body = ""
|
||||
bare = posting()
|
||||
bare.body = ""
|
||||
store.upsert_role(bare) # the title alone
|
||||
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(listed) # a teaser beats the title
|
||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||
role = store.role(listed.uid)
|
||||
assert role is not None and role.teaser == "Build the payments core."
|
||||
assert document(role) == "Senior Backend Engineer (m/w/d)\nBuild the payments core."
|
||||
assert store.roles_page()[0].excerpt == "Build the payments core."
|
||||
|
||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(bare) # a pull without it keeps it
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
|
||||
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
|
||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||
role = store.role(listed.uid)
|
||||
assert role is not None and document(role).endswith("We build things.")
|
||||
|
||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
|
||||
|
||||
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
|
||||
store.upsert_role(posting("1")) # "Senior Backend Engineer"
|
||||
uid = posting("1").uid
|
||||
|
||||
def levels() -> tuple[str, str, str, str]:
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
return (role.seniority, role.seniority_model, role.seniority_stated,
|
||||
role.seniority_title)
|
||||
|
||||
assert levels() == ("senior", "", "", "senior")
|
||||
store.upsert_role(posting("1", seniority="lead"))
|
||||
assert levels() == ("lead", "", "lead", "senior")
|
||||
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
|
||||
assert levels() == ("senior", "", "", "senior")
|
||||
store.set_extracted(uid, {"seniority": "staff"})
|
||||
store.upsert_role(posting("1", seniority="lead"))
|
||||
assert levels() == ("staff", "staff", "lead", "senior")
|
||||
|
||||
|
||||
def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> None:
|
||||
path = tmp_path / "v18.db"
|
||||
store = Store(path)
|
||||
joined = posting("joined", place="Copenhagen; Malmö")
|
||||
detailed = posting("detailed", place="Aarhus")
|
||||
stated = posting("stated", seniority="lead")
|
||||
extracted = posting("extracted")
|
||||
for item in (joined, detailed, stated, extracted):
|
||||
store.upsert_role(item)
|
||||
with sqlite3.connect(path) as conn:
|
||||
_undo_v19(conn)
|
||||
# A role whose own page was read is pulled with an empty place in raw.
|
||||
conn.execute("UPDATE roles SET raw = '{}' WHERE uid = ?", (detailed.uid,))
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-23', seniority = 'staff' "
|
||||
"WHERE uid = ?", (extracted.uid,))
|
||||
conn.execute("PRAGMA user_version = 18")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [19]
|
||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||
assert migrated.role_places(stated.uid) == []
|
||||
with sqlite3.connect(path) as conn:
|
||||
levels = {uid: rest for uid, *rest in conn.execute(
|
||||
"SELECT uid, seniority, seniority_model, seniority_stated, seniority_title, teaser "
|
||||
"FROM roles")}
|
||||
assert levels == {
|
||||
joined.uid: ["senior", "", "", "senior", ""],
|
||||
detailed.uid: ["senior", "", "", "senior", ""],
|
||||
stated.uid: ["lead", "", "lead", "senior", ""],
|
||||
extracted.uid: ["staff", "staff", "", "senior", ""],
|
||||
}
|
||||
|
||||
|
||||
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
||||
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
|
||||
salary_max=95000, currency="EUR", department="Eng"))
|
||||
@@ -738,12 +857,20 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
|
||||
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None:
|
||||
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
|
||||
store: Store) -> None:
|
||||
store.upsert_role(posting("1", seniority="junior"))
|
||||
uid = posting("1").uid
|
||||
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
|
||||
store.set_extracted(uid, {"seniority": "senior"})
|
||||
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
assert (role.seniority, role.seniority_stated, role.seniority_title,
|
||||
role.seniority_model) == ("junior", "junior", "senior", "")
|
||||
store.set_extracted(uid, {"seniority": "staff"})
|
||||
store.upsert_role(posting("1", seniority="junior"))
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
|
||||
("staff", "junior", "staff")
|
||||
|
||||
|
||||
def test_company_falls_back_to_the_subscription(store: Store) -> None:
|
||||
|
||||
Reference in new issue
Block a user