Places as a gazetteer, role_places, teaser and three seniority sources

Migration 19 adds places, place_aliases, place_includes (seeded from the
dictionaries in app/models.py and app/places.py, Europe as a continent over
the European countries) and role_places, one row per place of a role,
resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them
from raw["places"], else the place line cut at ";"; the migration builds them
from each role's place column.

The Place filter and the country and place gates are joins over the tree:
country:DK is any place under Denmark, region:skane any town in Skane, a
country gate is met by any place in its countries, and the EU fallback of
the word matcher is no longer used. A place: token the gazetteer does not
know still matches whole words. Store gains gazetteer, role_places,
unresolved_places, add_place_alias, add_place and relink_places.

roles.teaser is kept-filled and embedded while there is no description; a
vector is dropped when a better text arrives. seniority_model, _stated and
_title sit behind roles.seniority, which stays the first filled of them.

TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads
READS.fields.posted and falls back to the old set for undeclared readers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:31:45 +02:00
1 parent 4c0da58da7
commit bbd7522163
10 files changed
+1437 -98

No files matched your search

+139 -12
View File
@@ -14,8 +14,8 @@ from desk.models import Item
import numpy as np
from app.models import (
LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key,
posted_at, posted_host, title_key,
LEVELS, Application, Gate, Limits, Prompt, company_key, place_key,
posted_at, posted_host, title_key, trusted_posted,
)
from app.storage.store import pack
from app.storage import LATEST, Store
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 18
assert LATEST == 19
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -88,8 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
def _undo_v19(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v18 shape: no gazetteer, no role places,
no teaser, one level."""
for table in ("role_places", "place_includes", "place_aliases", "places"):
conn.execute(f"DROP TABLE {table}")
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
def _undo_v18(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v17 shape: no copies."""
_undo_v19(conn)
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
@@ -166,7 +176,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18]
assert migrated.migrations_applied == [15, 16, 17, 18, 19]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -371,7 +381,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -395,7 +405,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18]
assert migrated.migrations_applied == [16, 17, 18, 19]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -424,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18]
assert migrated.migrations_applied == [17, 18, 19]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -517,7 +527,7 @@ def test_posted_host_reads_board_then_provider() -> None:
def test_posted_at_trusts_only_a_known_host() -> None:
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS
assert trusted_posted("greenhouse") and not trusted_posted("taleo")
early = "2020-01-01T00:00:00+00:00"
late = "2026-09-23T09:00:00+00:00"
# A trusted host's earlier date wins over first_seen...
@@ -527,6 +537,29 @@ def test_posted_at_trusts_only_a_known_host() -> None:
assert posted_at(early, late, "") == late
def test_a_reader_that_declares_reads_decides_its_own_date_trust(
monkeypatch: pytest.MonkeyPatch) -> None:
"""`fields.posted` of a declared reader wins over the fallback set, either
way; a reader without `READS`, or a provider that is none, keeps it."""
import app.models as models
monkeypatch.setattr(models, "_declared_posted",
lambda: {"greenhouse": "listed", "gem": "trusted"})
assert not trusted_posted("greenhouse")
assert trusted_posted("gem")
assert trusted_posted("jobindex")
assert not trusted_posted("taleo")
def test_the_declarations_are_read_off_the_readers() -> None:
import app.models as models
from plugins.readers import READERS
declared = models._declared_posted()
assert set(declared) <= set(READERS)
assert all(value in ("trusted", "listed", "none") for value in declared.values())
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
first_seen = "2026-09-23T09:00:00+00:00"
future = "2026-09-24T00:00:00+00:00" # after first_seen
@@ -574,6 +607,92 @@ def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store)
assert store.roles_to_embed("m", 10) == []
def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
store: Store) -> None:
from app.jobs.embed import document
listed = posting(teaser="Build the payments core.")
listed.body = ""
bare = posting()
bare.body = ""
store.upsert_role(bare) # the title alone
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(listed) # a teaser beats the title
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and role.teaser == "Build the payments core."
assert document(role) == "Senior Backend Engineer (m/w/d)\nBuild the payments core."
assert store.roles_page()[0].excerpt == "Build the payments core."
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(bare) # a pull without it keeps it
assert store.roles_to_embed("m", 10) == []
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and document(role).endswith("We build things.")
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
assert store.roles_to_embed("m", 10) == []
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
store.upsert_role(posting("1")) # "Senior Backend Engineer"
uid = posting("1").uid
def levels() -> tuple[str, str, str, str]:
role = store.role(uid)
assert role is not None
return (role.seniority, role.seniority_model, role.seniority_stated,
role.seniority_title)
assert levels() == ("senior", "", "", "senior")
store.upsert_role(posting("1", seniority="lead"))
assert levels() == ("lead", "", "lead", "senior")
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
assert levels() == ("senior", "", "", "senior")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="lead"))
assert levels() == ("staff", "staff", "lead", "senior")
def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> None:
path = tmp_path / "v18.db"
store = Store(path)
joined = posting("joined", place="Copenhagen; Malmö")
detailed = posting("detailed", place="Aarhus")
stated = posting("stated", seniority="lead")
extracted = posting("extracted")
for item in (joined, detailed, stated, extracted):
store.upsert_role(item)
with sqlite3.connect(path) as conn:
_undo_v19(conn)
# A role whose own page was read is pulled with an empty place in raw.
conn.execute("UPDATE roles SET raw = '{}' WHERE uid = ?", (detailed.uid,))
conn.execute("UPDATE roles SET extracted_at = '2026-09-23', seniority = 'staff' "
"WHERE uid = ?", (extracted.uid,))
conn.execute("PRAGMA user_version = 18")
migrated = Store(path)
assert migrated.migrations_applied == [19]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
with sqlite3.connect(path) as conn:
levels = {uid: rest for uid, *rest in conn.execute(
"SELECT uid, seniority, seniority_model, seniority_stated, seniority_title, teaser "
"FROM roles")}
assert levels == {
joined.uid: ["senior", "", "", "senior", ""],
detailed.uid: ["senior", "", "", "senior", ""],
stated.uid: ["lead", "", "lead", "senior", ""],
extracted.uid: ["staff", "staff", "", "senior", ""],
}
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
salary_max=95000, currency="EUR", department="Eng"))
@@ -738,12 +857,20 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None:
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
store: Store) -> None:
store.upsert_role(posting("1", seniority="junior"))
uid = posting("1").uid
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
store.set_extracted(uid, {"seniority": "senior"})
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_title,
role.seniority_model) == ("junior", "junior", "senior", "")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="junior"))
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
("staff", "junior", "staff")
def test_company_falls_back_to_the_subscription(store: Store) -> None: