Filter the feed by company keys, place tokens and seniority, and suggest companies and places

Migration 17 adds roles.place_fold and fills seniority from the title where extract has not read the role. Every connection registers place_match, so the Place filter and the place gate match whole folded words: Lund no longer finds Billund. GET /api/roles takes company_keys, places, seniority, seniority_unknown and pay_unknown; new routes suggest companies and places, label place tokens and count the seniority and pay facets.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
WolfandClaude committed 2026-09-29 22:17:12 +02:00
1 parent 4b9403a0b9
commit a91b4e99e0
6 files changed
+724 -60

No files matched your search

+217 -7
View File
@@ -14,8 +14,8 @@ from desk.models import Item
import numpy as np
from app.models import (
Application, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at,
posted_host, title_key,
LEVELS, Application, Gate, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key,
posted_at, posted_host, title_key,
)
from app.storage.store import pack
from app.storage import LATEST, Store
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 16
assert LATEST == 17
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -88,9 +88,15 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
def _undo_v17(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v16 shape: no folded place on a role."""
conn.execute("ALTER TABLE roles DROP COLUMN place_fold")
def _undo_v16(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v15 shape: no by-hand reason on a
company."""
_undo_v17(conn)
conn.execute("ALTER TABLE companies DROP COLUMN hand_reason")
@@ -105,7 +111,7 @@ def _undo_v15(conn: sqlite3.Connection) -> None:
def _undo_v14(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v13 shape: an agent's intent and its
cached vector on the agent row, no prompts, no prompt on a score. Every
downgrade below starts here, after `_undo_v15`, since v16 is the last
downgrade below starts here, after `_undo_v15`, since v17 is the last
migration."""
_undo_v15(conn)
conn.execute("DROP TABLE agent_prompts")
@@ -153,7 +159,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16]
assert migrated.migrations_applied == [15, 16, 17]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -358,7 +364,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16]
assert migrated.migrations_applied == [14, 15, 16, 17]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -382,7 +388,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16]
assert migrated.migrations_applied == [16, 17]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -390,6 +396,35 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
assert sorted(_columns(path, "companies")) == sorted(_columns(fresh, "companies"))
def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
tmp_path: Path) -> None:
path = tmp_path / "v16.db"
store = Store(path)
plain = posting("plain", place="København K") # Senior ... in the title
extracted = posting("extracted", place="Malmö")
kept = posting("kept")
unknown = posting("unknown")
unknown.title = "Backend Engineer"
for item in (plain, extracted, kept, unknown):
store.upsert_role(item)
with sqlite3.connect(path) as conn:
_undo_v17(conn)
conn.execute("UPDATE roles SET seniority = ''")
conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?",
(extracted.uid,))
conn.execute("UPDATE roles SET seniority = 'lead' WHERE uid = ?", (kept.uid,))
conn.execute("PRAGMA user_version = 16")
migrated = Store(path)
assert migrated.migrations_applied == [17]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
assert rows == {plain.uid: ("kobenhavn k", "senior"), extracted.uid: ("malmo", ""),
kept.uid: ("", "lead"), unknown.uid: ("", "")}
def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None:
ai = Prompt(name="AI leadership", text="Head of AI")
fde = Prompt(name="Forward deployed", text="FDE lead")
@@ -639,6 +674,53 @@ def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Sto
assert role is not None and role.function == "sales"
def _folded(store: Store, uid: str) -> str:
with sqlite3.connect(store.path) as conn:
return str(conn.execute("SELECT place_fold FROM roles WHERE uid = ?",
(uid,)).fetchone()[0])
def test_place_fold_is_written_with_place(store: Store) -> None:
store.upsert_role(posting("1", place="København K"))
uid = posting("1").uid
assert _folded(store, uid) == "kobenhavn k"
store.upsert_role(posting("1")) # says less: both kept
assert (store.role(uid).place, _folded(store, uid)) == ( # type: ignore[union-attr]
"København K", "kobenhavn k")
store.upsert_role(posting("1", place="Kgs. Lyngby"))
assert _folded(store, uid) == "kgs lyngby"
store.upsert_role(posting("2"))
store.set_extracted(posting("2").uid, {"place": "Malmö, Sweden"})
assert _folded(store, posting("2").uid) == "malmo sweden"
def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> None:
store.upsert_role(posting("1")) # "Senior Backend Engineer"
uid = posting("1").uid
assert store.role(uid).seniority == "senior" # type: ignore[union-attr]
retitled = posting("1")
retitled.title = "Backend Engineer"
store.upsert_role(retitled) # the title no longer says
assert store.role(uid).seniority == "" # type: ignore[union-attr]
store.upsert_role(posting("1"))
store.set_extracted(uid, {"seniority": "lead"}) # extract overwrites the title's
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
store.upsert_role(posting("1")) # and a pull does not put it back
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
store.set_extracted(uid, {})
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
def test_a_level_the_board_states_wins_over_the_title_and_extract(store: Store) -> None:
store.upsert_role(posting("1", seniority="junior"))
uid = posting("1").uid
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
store.set_extracted(uid, {"seniority": "senior"})
assert store.role(uid).seniority == "junior" # type: ignore[union-attr]
def test_company_falls_back_to_the_subscription(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
store.upsert_role(posting(), sub)
@@ -749,6 +831,116 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No
store.roles(pay="bogus")
def test_roles_filter_by_company_keys(store: Store) -> None:
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Acme Labs"))
store.upsert_role(posting("3", company="Other"))
assert {r.source_id for r in store.roles(company_keys=["acme"])} == {"1"}
assert {r.source_id for r in store.roles(company_keys=["acme", "other"])} == {"1", "3"}
assert store.count_roles(company_keys=["acme-labs"]) == 1
assert store.roles(company_keys=["nobody"]) == []
def test_roles_filter_by_place_tokens_on_whole_words(store: Store) -> None:
for sid, place, country in (
("kbh", "København K", ""), ("ballerup", "Ballerup", ""),
("offices", "Copenhagen; London; Warsaw", ""), ("aarhus", "Aarhus C", "DK"),
("malmo", "Malmö", ""), ("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
("berlin", "Berlin", "")):
store.upsert_role(posting(sid, place=place, country=country))
def found(*tokens: str) -> set[str]:
return {r.source_id for r in store.roles(places=tokens)}
assert found("country:DK") == {"kbh", "ballerup", "offices", "aarhus", "billund"}
assert found("region:skane") == {"malmo", "lund"}
assert found("country:DK", "region:skane") == \
{"kbh", "ballerup", "offices", "aarhus", "billund", "malmo", "lund"}
assert found("place:lund") == {"lund"} # not Billund
assert found("Lund") == {"lund"} # bare text is a place
assert found("place:copenhagen") == {"kbh", "offices"} # and its aliases
assert store.count_roles(places=["region:skane"]) == 2
with pytest.raises(ValueError, match="region:atlantis"):
store.roles(places=["region:atlantis"])
def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store) -> None:
for sid, title in (("senior", "Senior Engineer"), ("junior", "Junior Engineer"),
("lead", "Tech Lead"), ("none", "Engineer")):
item = posting(sid)
item.title = title
store.upsert_role(item)
def found(*levels: str, unknown: bool = True) -> set[str]:
return {r.source_id for r in store.roles(seniority=levels, seniority_unknown=unknown)}
assert found() == {"senior", "junior", "lead", "none"}
assert found("senior") == {"senior", "none"}
assert found("senior", "lead") == {"senior", "lead", "none"}
assert found("senior", "lead", unknown=False) == {"senior", "lead"}
assert store.count_roles(seniority=["junior"], seniority_unknown=False) == 1
assert "intern" in LEVELS and "" not in LEVELS
with pytest.raises(ValueError, match="seniority is"):
store.roles(seniority=["boss"])
def test_roles_pay_unknown_lets_the_roles_with_no_pay_through(store: Store) -> None:
store.upsert_role(posting("normalised", salary_min=70000, currency="EUR"))
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("none"))
assert {r.source_id for r in store.roles(pay="normalised", pay_unknown=True)} == \
{"normalised", "none"}
assert {r.source_id for r in store.roles(pay="stated", pay_unknown=True)} == \
{"normalised", "stated", "none"}
assert store.count_roles(pay="normalised") == 1
assert store.count_roles(pay_unknown=True) == 3 # no pay filter, nothing to widen
def test_role_facets_count_each_facet_with_its_own_filter_left_out(store: Store) -> None:
for sid, title, raw in (
("a", "Senior Engineer", {"salary_min": 70000, "currency": "EUR"}),
("b", "Senior Engineer", {"salary_text": "USD 45 per hour"}),
("c", "Junior Engineer", {}),
("d", "Engineer", {}),
("e", "Senior Engineer", {"company": "Other"})):
item = posting(sid, **raw)
item.title = title
store.upsert_role(item)
facets = store.role_facets(seniority=["junior"], seniority_unknown=False,
pay="normalised")
# Seniority counted under the pay filter only: a.
assert dict(facets["seniority"]) == {**dict.fromkeys(LEVELS, 0), "senior": 1, "": 0}
# Pay counted under the seniority filter only: c.
assert facets["pay"] == [("normalised", 0), ("stated", 0), ("", 1)]
assert [v for v, _ in facets["seniority"]] == [*LEVELS, ""]
everything = store.role_facets()
assert dict(everything["seniority"])["senior"] == 3
assert dict(everything["seniority"])[""] == 1
assert everything["pay"] == [("normalised", 1), ("stated", 1), ("", 3)]
assert dict(store.role_facets(company_keys=["other"])["seniority"])["senior"] == 1
def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store) -> None:
# One key, three spellings: the company row's name first, then the roles'
# by how many carry each.
store.add_company("Ørsted Group", key="ørsted")
store.upsert_role(posting("1", company="Ørsted A/S", place="Gentofte"))
store.upsert_role(posting("2", company="Ørsted", place="Gentofte"))
store.upsert_role(posting("3", company="Ørsted", place="Malmö"))
store.upsert_role(posting("4", company="Hidden", place="Malmö"))
agent = store.save_agent("A")
store.set_affinity(agent.id, "hidden", -1)
store.set_affinity(agent.id, "ørsted", 2)
spellings = ["Ørsted Group", "Ørsted", "Ørsted A/S"]
assert sorted(store.open_companies()) == [
("hidden", ["Hidden"], 1, 0), ("ørsted", spellings, 3, 0)]
assert store.open_companies(agent.id) == [("ørsted", spellings, 3, 2)]
assert sorted(store.open_places()) == [("Gentofte", "", 2), ("Malmö", "SE", 2)]
def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None:
agent = store.save_agent("A")
store.add_company("Acme", key="acme")
@@ -1116,6 +1308,24 @@ def test_the_default_agent_is_the_lowest_enabled_id(store: Store) -> None:
assert store.default_agent() == later
def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None:
for sid, place, country in (("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
("kbh", "København K", ""), ("none", "", "")):
store.upsert_role(posting(sid, place=place, country=country))
agent = store.save_agent("A")
uid = {r.source_id: r.uid for r in store.roles()}
def states(*values: str) -> dict[str, int]:
found = store.score_candidates(agent.id, [Gate(field="place", values=list(values))])
return {sid: next(c.gates[0] for c in found if c.uid == u) for sid, u in uid.items()}
assert states("Lund") == {"lund": 1, "billund": 0, "kbh": 0, "none": -1}
assert states("Copenhagen", "region:skane") == {"lund": 1, "billund": 0, "kbh": 1,
"none": -1}
assert states("country:DK") == {"lund": 0, "billund": 1, "kbh": 1, "none": -1}
assert states("region:atlantis") == {"lund": 0, "billund": 0, "kbh": 0, "none": -1}
def test_open_role_counts_and_company_boards(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme")