Places as a gazetteer, role_places, teaser and three seniority sources
Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
bbd7522163
10 files changed
+1437
-98
No files matched your search
+139
-12
@@ -14,8 +14,8 @@ from desk.models import Item
|
||||
import numpy as np
|
||||
|
||||
from app.models import (
|
||||
LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key,
|
||||
posted_at, posted_host, title_key,
|
||||
LEVELS, Application, Gate, Limits, Prompt, company_key, place_key,
|
||||
posted_at, posted_host, title_key, trusted_posted,
|
||||
)
|
||||
from app.storage.store import pack
|
||||
from app.storage import LATEST, Store
|
||||
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 18
|
||||
assert LATEST == 19
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -88,8 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
|
||||
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
|
||||
|
||||
|
||||
def _undo_v19(conn: sqlite3.Connection) -> None:
|
||||
"""A fresh file put back to the v18 shape: no gazetteer, no role places,
|
||||
no teaser, one level."""
|
||||
for table in ("role_places", "place_includes", "place_aliases", "places"):
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
|
||||
|
||||
def _undo_v18(conn: sqlite3.Connection) -> None:
|
||||
"""A fresh file put back to the v17 shape: no copies."""
|
||||
_undo_v19(conn)
|
||||
conn.execute("DROP INDEX idx_roles_dup_of")
|
||||
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
|
||||
|
||||
@@ -166,7 +176,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18]
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||
@@ -371,7 +381,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18]
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19]
|
||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||
Prompt(name="Backend", text="python backend")]
|
||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||
@@ -395,7 +405,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [16, 17, 18]
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||
@@ -424,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [17, 18]
|
||||
assert migrated.migrations_applied == [17, 18, 19]
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||
"SELECT uid, place_fold, seniority FROM roles")}
|
||||
@@ -517,7 +527,7 @@ def test_posted_host_reads_board_then_provider() -> None:
|
||||
|
||||
|
||||
def test_posted_at_trusts_only_a_known_host() -> None:
|
||||
assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS
|
||||
assert trusted_posted("greenhouse") and not trusted_posted("taleo")
|
||||
early = "2020-01-01T00:00:00+00:00"
|
||||
late = "2026-09-23T09:00:00+00:00"
|
||||
# A trusted host's earlier date wins over first_seen...
|
||||
@@ -527,6 +537,29 @@ def test_posted_at_trusts_only_a_known_host() -> None:
|
||||
assert posted_at(early, late, "") == late
|
||||
|
||||
|
||||
def test_a_reader_that_declares_reads_decides_its_own_date_trust(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`fields.posted` of a declared reader wins over the fallback set, either
|
||||
way; a reader without `READS`, or a provider that is none, keeps it."""
|
||||
import app.models as models
|
||||
|
||||
monkeypatch.setattr(models, "_declared_posted",
|
||||
lambda: {"greenhouse": "listed", "gem": "trusted"})
|
||||
assert not trusted_posted("greenhouse")
|
||||
assert trusted_posted("gem")
|
||||
assert trusted_posted("jobindex")
|
||||
assert not trusted_posted("taleo")
|
||||
|
||||
|
||||
def test_the_declarations_are_read_off_the_readers() -> None:
|
||||
import app.models as models
|
||||
from plugins.readers import READERS
|
||||
|
||||
declared = models._declared_posted()
|
||||
assert set(declared) <= set(READERS)
|
||||
assert all(value in ("trusted", "listed", "none") for value in declared.values())
|
||||
|
||||
|
||||
def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
|
||||
first_seen = "2026-09-23T09:00:00+00:00"
|
||||
future = "2026-09-24T00:00:00+00:00" # after first_seen
|
||||
@@ -574,6 +607,92 @@ def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store)
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
|
||||
|
||||
def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
|
||||
store: Store) -> None:
|
||||
from app.jobs.embed import document
|
||||
|
||||
listed = posting(teaser="Build the payments core.")
|
||||
listed.body = ""
|
||||
bare = posting()
|
||||
bare.body = ""
|
||||
store.upsert_role(bare) # the title alone
|
||||
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(listed) # a teaser beats the title
|
||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||
role = store.role(listed.uid)
|
||||
assert role is not None and role.teaser == "Build the payments core."
|
||||
assert document(role) == "Senior Backend Engineer (m/w/d)\nBuild the payments core."
|
||||
assert store.roles_page()[0].excerpt == "Build the payments core."
|
||||
|
||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(bare) # a pull without it keeps it
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
|
||||
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
|
||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||
role = store.role(listed.uid)
|
||||
assert role is not None and document(role).endswith("We build things.")
|
||||
|
||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
|
||||
|
||||
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
|
||||
store.upsert_role(posting("1")) # "Senior Backend Engineer"
|
||||
uid = posting("1").uid
|
||||
|
||||
def levels() -> tuple[str, str, str, str]:
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
return (role.seniority, role.seniority_model, role.seniority_stated,
|
||||
role.seniority_title)
|
||||
|
||||
assert levels() == ("senior", "", "", "senior")
|
||||
store.upsert_role(posting("1", seniority="lead"))
|
||||
assert levels() == ("lead", "", "lead", "senior")
|
||||
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
|
||||
assert levels() == ("senior", "", "", "senior")
|
||||
store.set_extracted(uid, {"seniority": "staff"})
|
||||
store.upsert_role(posting("1", seniority="lead"))
|
||||
assert levels() == ("staff", "staff", "lead", "senior")
|
||||
|
||||
|
||||
def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> None:
|
||||
path = tmp_path / "v18.db"
|
||||
store = Store(path)
|
||||
joined = posting("joined", place="Copenhagen; Malmö")
|
||||
detailed = posting("detailed", place="Aarhus")
|
||||
stated = posting("stated", seniority="lead")
|
||||
extracted = posting("extracted")
|
||||
for item in (joined, detailed, stated, extracted):
|
||||
store.upsert_role(item)
|
||||
with sqlite3.connect(path) as conn:
|
||||
_undo_v19(conn)
|
||||
# A role whose own page was read is pulled with an empty place in raw.
|
||||
conn.execute("UPDATE roles SET raw = '{}' WHERE uid = ?", (detailed.uid,))
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-23', seniority = 'staff' "
|
||||
"WHERE uid = ?", (extracted.uid,))
|
||||
conn.execute("PRAGMA user_version = 18")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [19]
|
||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||
assert migrated.role_places(stated.uid) == []
|
||||
with sqlite3.connect(path) as conn:
|
||||
levels = {uid: rest for uid, *rest in conn.execute(
|
||||
"SELECT uid, seniority, seniority_model, seniority_stated, seniority_title, teaser "
|
||||
"FROM roles")}
|
||||
assert levels == {
|
||||
joined.uid: ["senior", "", "", "senior", ""],
|
||||
detailed.uid: ["senior", "", "", "senior", ""],
|
||||
stated.uid: ["lead", "", "lead", "senior", ""],
|
||||
extracted.uid: ["staff", "staff", "", "senior", ""],
|
||||
}
|
||||
|
||||
|
||||
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
||||
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
|
||||
salary_max=95000, currency="EUR", department="Eng"))
|
||||
@@ -738,12 +857,20 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
|
||||
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None:
|
||||
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
|
||||
store: Store) -> None:
|
||||
store.upsert_role(posting("1", seniority="junior"))
|
||||
uid = posting("1").uid
|
||||
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
|
||||
store.set_extracted(uid, {"seniority": "senior"})
|
||||
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
assert (role.seniority, role.seniority_stated, role.seniority_title,
|
||||
role.seniority_model) == ("junior", "junior", "senior", "")
|
||||
store.set_extracted(uid, {"seniority": "staff"})
|
||||
store.upsert_role(posting("1", seniority="junior"))
|
||||
role = store.role(uid)
|
||||
assert role is not None
|
||||
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
|
||||
("staff", "junior", "staff")
|
||||
|
||||
|
||||
def test_company_falls_back_to_the_subscription(store: Store) -> None:
|
||||
|
||||
Reference in new issue
Block a user