Places as a gazetteer, role_places, teaser and three seniority sources

Migration 19 adds places, place_aliases, place_includes (seeded from the
dictionaries in app/models.py and app/places.py, Europe as a continent over
the European countries) and role_places, one row per place of a role,
resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them
from raw["places"], else the place line cut at ";"; the migration builds them
from each role's place column.

The Place filter and the country and place gates are joins over the tree:
country:DK is any place under Denmark, region:skane any town in Skane, a
country gate is met by any place in its countries, and the EU fallback of
the word matcher is no longer used. A place: token the gazetteer does not
know still matches whole words. Store gains gazetteer, role_places,
unresolved_places, add_place_alias, add_place and relink_places.

roles.teaser is kept-filled and embedded while there is no description; a
vector is dropped when a better text arrives. seniority_model, _stated and
_title sit behind roles.seniority, which stays the first filled of them.

TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads
READS.fields.posted and falls back to the old set for undeclared readers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:31:45 +02:00
1 parent 4c0da58da7
commit bbd7522163
10 files changed
+1437 -98

No files matched your search

+5 -1
View File
@@ -164,13 +164,17 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
store.upsert_role(item)
store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))])
with sqlite3.connect(path) as conn:
for table in ("role_places", "place_includes", "place_aliases", "places"):
conn.execute(f"DROP TABLE {table}")
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
conn.execute("PRAGMA user_version = 17")
migrated = Store(path)
assert migrated.migrations_applied == [18]
assert migrated.migrations_applied == [18, 19]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+397
View File
@@ -0,0 +1,397 @@
"""The gazetteer: its seed, the resolver, and a role's places in the store.
The store tests go through `Store`, as the feed and the gates do. The
real-line test compares the join over the gazetteer with the whole-word rule
it replaced (`places.matches`) on the live lines `test_places` holds: the
join must find every role the word rule found, but for the lines listed where
the word rule was wrong.
"""
from __future__ import annotations
import sqlite3
from collections import Counter
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
import pytest
from desk.models import Item
from app import places
from app.models import Gate, _CITIES, _COUNTRY_NAMES
from app.places import Gazetteer, fold, seed
from app.storage import Store
from test_places import LINES, ORACLE_TOKENS
def gazetteer() -> Gazetteer:
seeds, includes = seed()
ids: dict[tuple[str, str], int] = {}
rows, aliases = [], []
for n, s in enumerate(seeds, 1):
ids[(s.kind, s.key)] = n
rows.append((n, s.kind, s.key, s.name, ids[s.parent] if s.parent else None))
aliases += [(a, n) for a in s.aliases]
return Gazetteer(rows, aliases, [(ids[a], ids[b]) for a, b in includes])
GAZ = gazetteer()
def names(text: str) -> list[str]:
return [GAZ.places[p].name for p in GAZ.resolve(text)]
# ---------------------------------------------------------------------- seed
def test_the_seed_holds_the_dictionaries_in_code() -> None:
seeds, includes = seed()
kinds = Counter(s.kind for s in seeds)
assert kinds["continent"] == 1
assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line)
keys = {(s.kind, s.key) for s in seeds}
assert {("region", key) for key in places.REGIONS} <= keys
assert {("region", "oresund"), ("region", "skane")} in [set(i) for i in includes]
folds = {a for s in seeds for a in s.aliases}
assert set(_CITIES) - {"region hovedstaden"} <= folds
assert {"kobenhavn", "kopenhamn", "copenhagen", "skane", "emea", "eu"} <= folds
def test_parents_come_before_their_children() -> None:
seen: set[tuple[str, str]] = set()
for s in seed()[0]:
assert s.parent is None or s.parent in seen, s
seen.add((s.kind, s.key))
def test_copenhagen_is_in_the_capital_region_in_denmark_in_europe() -> None:
copenhagen = GAZ.by_key[("city", "copenhagen")]
above = {GAZ.places[p].key for p in GAZ.ancestors(copenhagen)}
assert above == {"hovedstaden", "oresund", "DK", "europe"}
assert GAZ.country(copenhagen) == "DK"
assert GAZ.by_key[("country", "US")] not in GAZ.under({GAZ.by_key[("continent", "europe")]})
# ------------------------------------------------------------------ resolving
@pytest.mark.parametrize("text, expected", [
("Aarhus N, Region Central Jutland, Denmark", ["Aarhus"]),
("København K", ["Copenhagen"]),
("Copenhagen Office", ["Copenhagen"]),
("Malmö/Lund", ["Malmö", "Lund"]),
("Aalborg, Copenhagen, Kolding", ["Aalborg", "Copenhagen", "Kolding"]),
("Paderborn, Germany", ["Germany"]),
("Skåne län, Sweden", ["Skåne"]),
("London, Ontario", ["Ontario"]),
("Bergen op Zoom, Netherlands", ["Netherlands"]),
("Mexicali, BAJA CALIFORNIA, Mexico", ["Mexico"]),
("Paris, TX", ["United States"]),
("Remote (DE)", ["Germany"]),
("London OR Paris OR Germany", ["London", "Paris", "Germany"]),
("Allerød or Stockholm (SE)", ["Allerød", "Stockholm"]),
("Finland, København, Litauen, Polen, Sverige", ["Finland", "Copenhagen", "Sweden"]),
("Remote, EU or US", ["Europe", "United States"]),
("Home based - EMEA", ["Europe"]),
("Luxembourg", ["Luxembourg"]),
("Nordhavn og mulighed for hjemmearbejde", ["Nordhavn"]),
("Remote", []),
("Oberkochen", []),
("", []),
])
def test_a_text_resolves_to_the_places_it_names(text: str, expected: list[str]) -> None:
assert names(text) == expected
def test_luxembourg_the_town_is_in_luxembourg_the_country() -> None:
town = GAZ.by_key[("city", "luxembourg")]
assert GAZ.resolve("Luxembourg") == (town,)
assert GAZ.country(town) == "LU"
@pytest.mark.parametrize("text", ["Remote", "Hybrid", "Multiple Locations", "2 Locations",
"Remote job", "", " "])
def test_a_text_of_how_a_role_is_worked_is_no_place(text: str) -> None:
assert places.placeless(text)
@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Jena"])
def test_a_text_with_a_name_is_a_place(text: str) -> None:
assert not places.placeless(text)
# -------------------------------------------------------------------- tokens
def test_a_token_names_places_before_what_lies_in_them() -> None:
assert GAZ.lookup("country:DK") == {GAZ.by_key[("country", "DK")]}
assert GAZ.lookup("country:EU") == {GAZ.by_key[("continent", "europe")]}
assert GAZ.lookup("region:Skåne") == {GAZ.by_key[("region", "skane")]}
assert GAZ.lookup("place:kobenhavn") == {GAZ.by_key[("city", "copenhagen")]}
assert GAZ.lookup("place:denmark") == {GAZ.by_key[("country", "DK")]}
assert GAZ.lookup("place:oberkochen") is None
assert GAZ.lookup("country:ZZ") is None
def test_europe_is_every_european_country_and_eu() -> None:
under = GAZ.under(GAZ.lookup("country:EU") or ())
codes = GAZ.codes(under)
assert {"DK", "SE", "DE", "GB", "CH", "EU"} <= codes
assert not {"US", "CN"} & codes
def test_oresund_holds_both_capital_regions() -> None:
under = GAZ.under(GAZ.lookup("region:oresund") or ())
assert {GAZ.by_key[("city", "malmo")], GAZ.by_key[("city", "copenhagen")]} <= under
assert GAZ.by_key[("city", "aarhus")] not in under
# --------------------------------------------------------------------- store
@pytest.fixture
def store(tmp_path: Path) -> Iterator[Store]:
yield Store(tmp_path / "roledesk.db")
def posting(source_id: str, *, title: str = "Backend Engineer", body: str = "",
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw=dict(raw))
item.source = "ats"
return item
def texts(store: Store, uid: str) -> list[tuple[int, str, str]]:
return [(p.position, p.text, p.name) for p in store.role_places(uid)]
def test_a_fresh_file_is_seeded(store: Store) -> None:
gaz = store.gazetteer()
assert len(gaz.places) == len(GAZ.places)
assert ("city", "copenhagen") in gaz.by_key
def test_a_joined_line_is_several_places(store: Store) -> None:
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
uid = posting("1").uid
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
(2, "Warsaw", "Warsaw")]
role = store.role(uid)
assert role is not None and role.place == "Copenhagen; London; Warsaw"
assert [p.name for p in role.places] == ["Copenhagen", "London", "Warsaw"]
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
store.upsert_role(posting("1", place="Berlin, Munich", places=[
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
{"text": "Remote (DE)", "remote": True}]))
assert texts(store, posting("1").uid) == [
(0, "Berlin", "Berlin"), (1, "Munich", "Munich"), (2, "Remote (DE)", "Germany")]
def test_places_without_a_line_make_the_line(store: Store) -> None:
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
{"text": "Malmö", "remote": None}]))
role = store.role(posting("1").uid)
assert role is not None
assert (role.place, role.country) == ("Lund; Malmö", "SE")
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
store.upsert_role(posting("1", place="Aarhus"))
store.upsert_role(posting("1"))
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
store.upsert_role(posting("1", place="Odense"))
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
store.upsert_role(posting("1", place="Aarhus"))
uid = posting("1").uid
with sqlite3.connect(store.path) as conn:
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
store.upsert_role(posting("1", place="Aalborg"))
assert [(p.text, p.url) for p in store.role_places(uid)] == [
("Aalborg", ""), ("Odense", "https://copy")]
def test_an_extracted_place_is_resolved(store: Store) -> None:
store.upsert_role(posting("1"))
uid = posting("1").uid
store.set_extracted(uid, {"place": "Lund, Sweden"})
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
# ---------------------------------------------------------- filter and gates
def uids(store: Store, *tokens: str) -> set[str]:
return {r.uid for r in store.roles(limit=100_000, places=list(tokens), by_score=False)}
@pytest.fixture
def several(store: Store) -> dict[str, str]:
lines = {"cph": "Copenhagen; London", "eu": "Berlin, Germany; Paris, France",
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
for key, line in lines.items():
store.upsert_role(posting(key, place=line))
return {key: posting(key).uid for key in lines}
def test_a_country_is_every_place_in_it(store: Store, several: dict[str, str]) -> None:
assert uids(store, "country:DK") == {several["cph"], several["billund"]}
assert uids(store, "country:SE") == {several["malmo"], several["lund"], several["remote"]}
assert uids(store, "country:DE") == {several["eu"]}
assert uids(store, "country:GB") == {several["cph"]}
def test_europe_is_every_european_role(store: Store, several: dict[str, str]) -> None:
assert uids(store, "country:EU") == set(several.values()) - {several["ny"], several["zeiss"]}
def test_a_region_is_every_town_in_it(store: Store, several: dict[str, str]) -> None:
assert uids(store, "region:skane") == {several["malmo"], several["lund"]}
assert uids(store, "region:oresund") == {several["malmo"], several["lund"], several["cph"]}
assert uids(store, "place:skane") == uids(store, "region:skane")
def test_a_town_is_its_place_and_lund_is_not_in_billund(store: Store,
several: dict[str, str]) -> None:
assert uids(store, "place:lund") == {several["lund"]}
assert uids(store, "place:kobenhavn") == {several["cph"]}
def test_a_town_the_gazetteer_lacks_is_found_by_its_words(store: Store,
several: dict[str, str]) -> None:
assert uids(store, "place:oberkochen") == {several["zeiss"]}
def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
with pytest.raises(ValueError):
store.roles(places=["region:atlantis"])
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
store.upsert_role(posting("1", country="DK"))
assert uids(store, "country:DK") == {posting("1").uid}
def gates(store: Store, gate: Gate) -> dict[str, int]:
agent = store.save_agent("A", gates=[gate])
return {c.uid: c.gates[0] for c in store.score_candidates(agent.id, agent.gates)}
def test_a_country_gate_is_met_by_any_place_in_the_country(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="country", values=["DK", "SE"], mode="any"))
assert {uid for uid, g in found.items() if g == 1} == {
several[k] for k in ("cph", "billund", "malmo", "lund", "remote")}
# Only a role with neither a country nor a known place has no evidence.
assert {uid for uid, g in found.items() if g == -1} == {several["zeiss"]}
assert found[several["ny"]] == 0 and found[several["eu"]] == 0
def test_the_eu_value_of_a_country_gate_is_europe(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="country", values=["EU"]))
assert found[several["eu"]] == 1 and found[several["cph"]] == 1
assert found[several["ny"]] == 0
def test_a_place_gate_matches_over_the_gazetteer(store: Store,
several: dict[str, str]) -> None:
found = gates(store, Gate(field="place", values=["Malmö", "Lund", "oberkochen"],
mode="any"))
assert {uid for uid, g in found.items() if g == 1} == {
several["malmo"], several["lund"], several["zeiss"]}
# ------------------------------------------------------------ unresolved texts
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
store.upsert_role(posting(str(n), place=line))
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
("jena", "Jena", 1)]
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
def test_an_alias_links_what_it_names(store: Store) -> None:
store.upsert_role(posting("1", place="Kbh. K"))
store.upsert_role(posting("2", place="Copenhagen"))
uid = posting("1").uid
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
assert texts(store, uid) == [(0, "Kbh. K", "Copenhagen")]
assert store.unresolved_places() == []
assert uids(store, "country:DK") == {uid, posting("2").uid}
# The second store, another process, sees the new alias too.
assert Store(store.path).gazetteer().lookup("place:kbh") == \
store.gazetteer().lookup("place:copenhagen")
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
store.upsert_role(posting("1", place="Oberkochen; Jena"))
store.upsert_role(posting("2", place="Paderborn, Germany"))
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
store.add_place("Paderborn", "city", "country:DE")
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
assert texts(store, posting("2").uid) == [(0, "Paderborn, Germany", "Paderborn")]
assert uids(store, "place:oberkochen") == {posting("1").uid}
assert store.gazetteer().country(oberkochen) == "DE"
with pytest.raises(ValueError):
store.add_place("Oberkochen", "city", "country:DE")
with pytest.raises(ValueError):
store.add_place("Atlantis", "country", "country:EU")
def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
with pytest.raises(ValueError):
store.add_place_alias("Kbh", "place:atlantis")
with pytest.raises(ValueError):
store.add_place_alias("!!", "place:copenhagen")
def test_relink_places_resolves_every_role_again(store: Store) -> None:
store.upsert_role(posting("1", place="Oberkochen"))
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE role_places SET place_id = "
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
assert store.relink_places() == 1
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "")]
# --------------------------------------------------------- against the words
#: Lines of `test_places.LINES` the word rule found for a token and the
#: gazetteer does not, each rightly: the London in Canada, and a town named
#: after Zurich in Illinois.
NOT_FOUND_ANY_MORE: dict[str, set[str]] = {
"country:GB": {"London, Canada; Remote, United States; Oak Brook, United States"},
"place:london": {"London, Canada; Remote, United States; Oak Brook, United States"},
"place:zurich": {"Lake Zurich, Illinois, United States"},
}
@pytest.fixture(scope="module")
def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tuple[str, str]]]:
store = Store(tmp_path_factory.mktemp("real") / "roledesk.db")
lines = {}
for n, (place, country) in enumerate(LINES):
item = posting(f"line{n}", place=place, country=country)
store.upsert_role(item)
lines[item.uid] = (place, country)
return store, lines
@pytest.mark.parametrize("token", ORACLE_TOKENS)
def test_the_gazetteer_finds_what_the_words_found(
real: tuple[Store, dict[str, tuple[str, str]]], token: str) -> None:
store, lines = real
by_words = {uid for uid, (place, country) in lines.items()
if places.matches(fold(place), country, token)}
by_places = uids(store, token)
lost = {lines[uid][0] for uid in by_words - by_places}
assert lost <= NOT_FOUND_ANY_MORE.get(token, set()), (token, lost)
+139 -12
View File
@@ -14,8 +14,8 @@ from desk.models import Item
import numpy as np
from app.models import (
LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key,
posted_at, posted_host, title_key,
LEVELS, Application, Gate, Limits, Prompt, company_key, place_key,
posted_at, posted_host, title_key, trusted_posted,
)
from app.storage.store import pack
from app.storage import LATEST, Store
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 18
assert LATEST == 19
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -88,8 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
def _undo_v19(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v18 shape: no gazetteer, no role places,
no teaser, one level."""
for table in ("role_places", "place_includes", "place_aliases", "places"):
conn.execute(f"DROP TABLE {table}")
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
def _undo_v18(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v17 shape: no copies."""
_undo_v19(conn)
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
@@ -166,7 +176,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18]
assert migrated.migrations_applied == [15, 16, 17, 18, 19]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -371,7 +381,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -395,7 +405,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18]
assert migrated.migrations_applied == [16, 17, 18, 19]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -424,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18]
assert migrated.migrations_applied == [17, 18, 19]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -517,7 +527,7 @@ def test_posted_host_reads_board_then_provider() -> None:
def test_posted_at_trusts_only_a_known_host() -> None:
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS
assert trusted_posted("greenhouse") and not trusted_posted("taleo")
early = "2020-01-01T00:00:00+00:00"
late = "2026-09-23T09:00:00+00:00"
# A trusted host's earlier date wins over first_seen...
@@ -527,6 +537,29 @@ def test_posted_at_trusts_only_a_known_host() -> None:
assert posted_at(early, late, "") == late
def test_a_reader_that_declares_reads_decides_its_own_date_trust(
monkeypatch: pytest.MonkeyPatch) -> None:
"""`fields.posted` of a declared reader wins over the fallback set, either
way; a reader without `READS`, or a provider that is none, keeps it."""
import app.models as models
monkeypatch.setattr(models, "_declared_posted",
lambda: {"greenhouse": "listed", "gem": "trusted"})
assert not trusted_posted("greenhouse")
assert trusted_posted("gem")
assert trusted_posted("jobindex")
assert not trusted_posted("taleo")
def test_the_declarations_are_read_off_the_readers() -> None:
import app.models as models
from plugins.readers import READERS
declared = models._declared_posted()
assert set(declared) <= set(READERS)
assert all(value in ("trusted", "listed", "none") for value in declared.values())
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
first_seen = "2026-09-23T09:00:00+00:00"
future = "2026-09-24T00:00:00+00:00" # after first_seen
@@ -574,6 +607,92 @@ def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store)
assert store.roles_to_embed("m", 10) == []
def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
store: Store) -> None:
from app.jobs.embed import document
listed = posting(teaser="Build the payments core.")
listed.body = ""
bare = posting()
bare.body = ""
store.upsert_role(bare) # the title alone
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(listed) # a teaser beats the title
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and role.teaser == "Build the payments core."
assert document(role) == "Senior Backend Engineer (m/w/d)\nBuild the payments core."
assert store.roles_page()[0].excerpt == "Build the payments core."
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(bare) # a pull without it keeps it
assert store.roles_to_embed("m", 10) == []
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and document(role).endswith("We build things.")
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
assert store.roles_to_embed("m", 10) == []
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
store.upsert_role(posting("1")) # "Senior Backend Engineer"
uid = posting("1").uid
def levels() -> tuple[str, str, str, str]:
role = store.role(uid)
assert role is not None
return (role.seniority, role.seniority_model, role.seniority_stated,
role.seniority_title)
assert levels() == ("senior", "", "", "senior")
store.upsert_role(posting("1", seniority="lead"))
assert levels() == ("lead", "", "lead", "senior")
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
assert levels() == ("senior", "", "", "senior")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="lead"))
assert levels() == ("staff", "staff", "lead", "senior")
def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> None:
path = tmp_path / "v18.db"
store = Store(path)
joined = posting("joined", place="Copenhagen; Malmö")
detailed = posting("detailed", place="Aarhus")
stated = posting("stated", seniority="lead")
extracted = posting("extracted")
for item in (joined, detailed, stated, extracted):
store.upsert_role(item)
with sqlite3.connect(path) as conn:
_undo_v19(conn)
# A role whose own page was read is pulled with an empty place in raw.
conn.execute("UPDATE roles SET raw = '{}' WHERE uid = ?", (detailed.uid,))
conn.execute("UPDATE roles SET extracted_at = '2026-09-23', seniority = 'staff' "
"WHERE uid = ?", (extracted.uid,))
conn.execute("PRAGMA user_version = 18")
migrated = Store(path)
assert migrated.migrations_applied == [19]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
with sqlite3.connect(path) as conn:
levels = {uid: rest for uid, *rest in conn.execute(
"SELECT uid, seniority, seniority_model, seniority_stated, seniority_title, teaser "
"FROM roles")}
assert levels == {
joined.uid: ["senior", "", "", "senior", ""],
detailed.uid: ["senior", "", "", "senior", ""],
stated.uid: ["lead", "", "lead", "senior", ""],
extracted.uid: ["staff", "staff", "", "senior", ""],
}
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
salary_max=95000, currency="EUR", department="Eng"))
@@ -738,12 +857,20 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None:
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
store: Store) -> None:
store.upsert_role(posting("1", seniority="junior"))
uid = posting("1").uid
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
store.set_extracted(uid, {"seniority": "senior"})
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_title,
role.seniority_model) == ("junior", "junior", "senior", "")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="junior"))
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
("staff", "junior", "staff")
def test_company_falls_back_to_the_subscription(store: Store) -> None: