Prep stage: host words stored raw, mapped centrally in app/prep after every pull

upsert_role stores the host's own words (roles.*_host), the place texts
unresolved when they change, and the company key unresolved on insert, and
marks the role prep_due. app.prep maps seniority, employment, remote and
country through the host tables in app/prep/hosts, resolves places and
company aliases, lists unknown words in `unmapped`, remaps every open role
when the tables' fingerprint changes, and runs rebuild_dupes, which moves
out of the pull's settle. Migration 22 adds the columns and the table and
fills the host words from raw. Cascade: pull, prep, embed, score, enrich,
prep, embed, score, extract, score, summarise. `app prep [unmapped]
[--remap]`, /api/prep/unmapped and a fold on the Sources page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:19:22 +02:00
1 parent a66aac95cf
commit d04ceae7e0
31 files changed
+1468 -330

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
"""`upsert`: a role stored as a pull stores it and then mapped, as the prep
stage after the pull maps it. For tests of what the feed, the gates and the
pages read, which are prep's values, not the raw ones `upsert_role` keeps."""
from __future__ import annotations
from typing import Any
from app import prep
from app.storage import Store
def upsert(store: Store, item: Any, subscription: Any = None) -> bool:
new = store.upsert_role(item, subscription)
prep.run(store, dupes=False)
return new
+1 -1
View File
@@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1]
#: no database, so storage may use them. `main` assembles the app from
#: everything below it; `cli` and `__main__` are the entry points and may
#: reach anything.
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli",
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "prep", "jobs", "web", "main", "cli",
"__main__"]
APP = "app"
EXTERNAL = {"desk", "pipeline", "auth"}
+12 -11
View File
@@ -13,6 +13,7 @@ from app.models import company_fold
from app.storage import Store
from app.storage.schema import _add_company_aliases
from desk.models import Item
from prepped import upsert
def undo_v20(conn: sqlite3.Connection) -> None:
@@ -56,10 +57,10 @@ def test_a_company_is_its_own_alias(store: Store) -> None:
def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(posting("1", "Orsted"))
store.upsert_role(posting("2", "ORSTED A/S"))
store.upsert_role(posting("3", "Ørsted"))
store.upsert_role(posting("4", "Nobody"))
upsert(store, posting("1", "Orsted"))
upsert(store, posting("2", "ORSTED A/S"))
upsert(store, posting("3", "Ørsted"))
upsert(store, posting("4", "Nobody"))
assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"]
assert [c.key for c in store.companies()] == ["ørsted"]
@@ -67,7 +68,7 @@ def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
def test_an_added_alias_finds_the_company(store: Store) -> None:
store.add_company("Ørsted")
store.add_alias("ørsted", "Dong Energy")
store.upsert_role(posting("1", "Dong Energy A/S"))
upsert(store, posting("1", "Dong Energy A/S"))
assert company_of(store, "1") == "ørsted"
with pytest.raises(KeyError):
store.add_alias("nope", "X")
@@ -90,15 +91,15 @@ def test_duplicates_are_listed_by_fold_and_not_merged(store: Store) -> None:
assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(store.companies()) == 3
# The company with the name's own key wins when a role names it.
store.upsert_role(posting("1", "Orsted"))
upsert(store, posting("1", "Orsted"))
assert company_of(store, "1") == "orsted"
def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
store.add_company("Ørsted")
store.add_company("Orsted")
store.upsert_role(posting("1", "Orsted"))
store.upsert_role(posting("2", "Ørsted"))
upsert(store, posting("1", "Orsted"))
upsert(store, posting("2", "Ørsted"))
sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted",
board="greenhouse")
a, b = store.save_agent("A"), store.save_agent("B")
@@ -116,7 +117,7 @@ def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
assert store.duplicate_companies() == []
assert store.company_aliases() == {"ørsted": ["orsted"]}
# A later role from the gone spelling lands on the kept company.
store.upsert_role(posting("3", "Orsted"))
upsert(store, posting("3", "Orsted"))
assert company_of(store, "3") == "ørsted"
with pytest.raises(ValueError):
store.merge_companies("ørsted", "ørsted")
@@ -147,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21]
assert reopened.migrations_applied == [20, 21, 22]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -184,7 +185,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_roles")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21]
assert reopened.migrations_applied == [21, 22]
assert {c.key: c.all_roles for c in reopened.companies()} == {
"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
params = {s.target: s.params for s in reopened.subscriptions(source="ats")}
+1 -1
View File
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21]
assert migrated.migrations_applied == [18, 19, 20, 21, 22]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+9 -7
View File
@@ -250,16 +250,16 @@ class _Context:
return t
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_enrich() -> None:
context = _Context()
empty = Tally(results=({"filled": 0},))
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 4},
context=context)
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
assert out["next"] == "extract" and context.made[0].carries["at"] == 8
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
{"cascade": "pull", "task": "enrich", "at": 4}, context=context)
assert out["next"] == "prep" and context.made[1].carries["at"] == 5
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
context=context)
@@ -290,8 +290,10 @@ def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
assert started is not None
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
key=lambda j: j.id)
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == \
["enrich", "prep", "embed", "score"]
assert [j.kind for j in jobs if j.kind != FINAL] == \
["enrich", enrich.KIND, "prep", "embed", "score"]
assert web["asked"] == [f"{BASE}/acme/1"]
embedded = next(j for j in jobs if j.kind == "embed")
assert embedded.result["embedded"] == 1
+15 -14
View File
@@ -16,6 +16,7 @@ from app.config import Config
from app.jobs import extract
from app.models import Limits
from app.storage import Store
from prepped import upsert
def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item:
@@ -77,7 +78,7 @@ def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
def test_a_batch_writes_the_columns_and_stamps_extracted_at(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
uid = posting("1").uid
store.upsert_role(posting("1"))
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
@@ -100,7 +101,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at(
def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
store.upsert_role(item)
upsert(store, item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
@@ -116,13 +117,13 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
def test_the_next_upsert_keeps_what_extract_filled(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
for r in roles])
modeltask.run(extract.TASK, config, store)
store.upsert_role(posting("1"))
upsert(store, posting("1"))
role = store.role(posting("1").uid)
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
@@ -131,7 +132,7 @@ def test_the_next_upsert_keeps_what_extract_filled(
def test_closed_and_answered_roles_are_not_offered(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
for sid in ("open", "closed", "done"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
ranked(store, posting("open"), posting("closed"), posting("done"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
@@ -147,7 +148,7 @@ def test_closed_and_answered_roles_are_not_offered(
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
@@ -170,7 +171,7 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
the boards produced. A role nobody ranks is never sent; a batch dealt a
uid finishes it."""
for sid in ("best", "second", "low", "unscored", "gated"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
one = store.save_agent("One", "python backend",
limits=Limits(extract_top=1, extract_min=0.5))
two = store.save_agent("Two", "python backend", enabled=False)
@@ -209,7 +210,7 @@ def test_a_value_outside_the_vocabulary_is_dropped() -> None:
def test_disabled_finds_nothing_and_asks_nobody(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
model = stub(monkeypatch, lambda roles: [])
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
@@ -220,7 +221,7 @@ def test_disabled_finds_nothing_and_asks_nobody(
def test_a_role_read_before_without_a_level_is_read_again_for_it(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", seniority="senior")
store.upsert_role(item)
upsert(store, item)
ranked(store, item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
@@ -243,8 +244,8 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
withtext = posting("text", title="Senior Engineer", seniority="mid")
noText = posting("teaser", title="Analyst")
noText.body = ""
store.upsert_role(withtext)
store.upsert_role(noText)
upsert(store, withtext)
upsert(store, noText)
with store.transaction() as conn:
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
"WHERE source_id = 'teaser'")
@@ -265,7 +266,7 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
def test_an_answer_with_no_level_is_charged_until_given_up(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1")
store.upsert_role(item)
upsert(store, item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
@@ -279,7 +280,7 @@ def test_an_answer_with_no_level_is_charged_until_given_up(
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
for item in items:
store.upsert_role(item)
upsert(store, item)
one = store.save_agent("One", "python backend")
two = store.save_agent("Two", "python backend")
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
@@ -296,7 +297,7 @@ def test_agreement_per_host(store: Store) -> None:
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
("5", "lever", "senior", "")]:
store.upsert_role(posting(sid, board=board, seniority=stated))
upsert(store, posting(sid, board=board, seniority=stated))
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
(model, sid))
+19 -18
View File
@@ -24,6 +24,7 @@ from app.places import Gazetteer, fold, seed
from app.storage import Store
from test_places import LINES, ORACLE_TOKENS
from prepped import upsert
def gazetteer() -> Gazetteer:
@@ -173,7 +174,7 @@ def test_a_fresh_file_is_seeded(store: Store) -> None:
def test_a_joined_line_is_several_places(store: Store) -> None:
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
upsert(store, posting("1", place="Copenhagen; London; Warsaw"))
uid = posting("1").uid
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
(2, "Warsaw", "Warsaw")]
@@ -183,7 +184,7 @@ def test_a_joined_line_is_several_places(store: Store) -> None:
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
store.upsert_role(posting("1", place="Berlin, Munich", places=[
upsert(store, posting("1", place="Berlin, Munich", places=[
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
{"text": "Remote (DE)", "remote": True}]))
assert texts(store, posting("1").uid) == [
@@ -191,7 +192,7 @@ def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
def test_places_without_a_line_make_the_line(store: Store) -> None:
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
upsert(store, posting("1", places=[{"text": "Lund", "remote": None},
{"text": "Malmö", "remote": None}]))
role = store.role(posting("1").uid)
assert role is not None
@@ -199,26 +200,26 @@ def test_places_without_a_line_make_the_line(store: Store) -> None:
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
store.upsert_role(posting("1", place="Aarhus"))
store.upsert_role(posting("1"))
upsert(store, posting("1", place="Aarhus"))
upsert(store, posting("1"))
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
store.upsert_role(posting("1", place="Odense"))
upsert(store, posting("1", place="Odense"))
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
store.upsert_role(posting("1", place="Aarhus"))
upsert(store, posting("1", place="Aarhus"))
uid = posting("1").uid
with sqlite3.connect(store.path) as conn:
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
store.upsert_role(posting("1", place="Aalborg"))
upsert(store, posting("1", place="Aalborg"))
assert [(p.text, p.url) for p in store.role_places(uid)] == [
("Aalborg", ""), ("Odense", "https://copy")]
def test_an_extracted_place_is_resolved(store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
store.set_extracted(uid, {"place": "Lund, Sweden"})
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
@@ -236,7 +237,7 @@ def several(store: Store) -> dict[str, str]:
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
for key, line in lines.items():
store.upsert_role(posting(key, place=line))
upsert(store, posting(key, place=line))
return {key: posting(key).uid for key in lines}
@@ -274,7 +275,7 @@ def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
store.upsert_role(posting("1", country="DK"))
upsert(store, posting("1", country="DK"))
assert uids(store, "country:DK") == {posting("1").uid}
@@ -312,15 +313,15 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store,
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
store.upsert_role(posting(str(n), place=line))
upsert(store, posting(str(n), place=line))
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
("jena", "Jena", 1)]
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
def test_an_alias_links_what_it_names(store: Store) -> None:
store.upsert_role(posting("1", place="Kbh. K"))
store.upsert_role(posting("2", place="Copenhagen"))
upsert(store, posting("1", place="Kbh. K"))
upsert(store, posting("2", place="Copenhagen"))
uid = posting("1").uid
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
@@ -333,8 +334,8 @@ def test_an_alias_links_what_it_names(store: Store) -> None:
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
store.upsert_role(posting("1", place="Oberkochen; Jena"))
store.upsert_role(posting("2", place="Paderborn, Germany"))
upsert(store, posting("1", place="Oberkochen; Jena"))
upsert(store, posting("2", place="Paderborn, Germany"))
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
store.add_place("Paderborn", "city", "country:DE")
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
@@ -355,7 +356,7 @@ def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
def test_relink_places_resolves_every_role_again(store: Store) -> None:
store.upsert_role(posting("1", place="Oberkochen"))
upsert(store, posting("1", place="Oberkochen"))
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE role_places SET place_id = "
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
@@ -381,7 +382,7 @@ def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tup
lines = {}
for n, (place, country) in enumerate(LINES):
item = posting(f"line{n}", place=place, country=country)
store.upsert_role(item)
upsert(store, item)
lines[item.uid] = (place, country)
return store, lines
+358
View File
@@ -0,0 +1,358 @@
"""`app.prep`: the host's words mapped centrally, the stage over the due
roles, the unmapped list, the table fingerprint and migration 22. The host
tables here are fakes; each real table is held to its host by R1's and R2's
tests, and to the vocabularies by the last tests of this file."""
from __future__ import annotations
import importlib
import json
import sqlite3
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import cli, models, prep # noqa: E402
from app.prep import hosts, words # noqa: E402
from app.prep.country import country, of_host # noqa: E402
from app.storage import Store # noqa: E402
from desk.models import Item # noqa: E402
from plugins.readers import _shared # noqa: E402
TABLES: dict[str, Any] = {
"acme": {
"seniority": {"experienced": "", "sr": "senior"},
"employment": {"regular full time": "full_time", "permanent": ""},
"remote": {"2": "hybrid", "fully": "remote"},
},
"codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}},
"parts": {"match": {"mode": "substring",
"rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}},
}
# ------------------------------------------------------------------- words
@pytest.mark.parametrize(("field", "word", "value"), [
("seniority", "SR", "senior"),
("seniority", " Experienced ", ""), # known, names no level
("seniority", "senior", "senior"), # already the vocabulary
("seniority", "Grand Poobah", None), # unknown
("employment", "Regular Full Time", "full_time"),
("employment", "Permanent", ""),
("employment", "contract", "contract"),
("employment", "Zero hours", None),
("remote", "2", "hybrid"),
("remote", "On-site", "onsite"), # the shared words
("remote", "on_site", "onsite"),
("remote", "IN_OFFICE", "onsite"),
("remote", "office", "onsite"),
("remote", "Hybrid", "hybrid"),
("remote", "unspecified", None),
("seniority", "", ""),
])
def test_a_word_goes_through_the_hosts_table_then_the_vocabulary(
field: str, word: str, value: str | None) -> None:
assert words.lookup(field, "acme", word, TABLES) == value
def test_another_hosts_table_is_not_used() -> None:
assert words.lookup("seniority", "other", "sr", TABLES) is None
def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None:
assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time"
assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time"
assert words.lookup("employment", "codes", "software", TABLES) is None
assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary"
assert words.employment("parts", "permanent_full_time", TABLES) == "full_time"
def test_seniority_is_the_stated_level_and_the_titles() -> None:
assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff")
assert words.seniority("acme", "", "Engineer", TABLES) == ("", "")
def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None:
assert words.remote("acme", True, (), "", TABLES) == "remote"
assert words.remote("acme", "true", (), "", TABLES) == "remote"
assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote"
# False says nothing the places do not: hosts send it for hybrid roles too.
assert words.remote("acme", False, (), "Berlin", TABLES) == ""
assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid"
assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \
== "remote"
assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}],
"", TABLES) == "remote"
# ----------------------------------------------------------------- country
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def ids(store: Store, *texts: str) -> list[int]:
gaz = store.gazetteer()
return [p for text in texts for p in gaz.resolve(text)]
def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None:
gaz = store.gazetteer()
assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK"
assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU"
assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == ""
assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB"
assert country(gaz, ids(store, "Europe")) == "EU"
assert country(gaz, [], "dk") == "DK"
assert country(gaz, ids(store, "Remote"), "Sverige") == "SE"
@pytest.mark.parametrize(("stated", "code"), [
("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"),
("", ""), ("Narnia", None)])
def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str,
code: str | None) -> None:
assert of_host(store.gazetteer(), stated) == code
# ------------------------------------------------------------------- stage
def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme",
**raw: Any) -> Item:
found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}",
title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"board": host, "company": "Acme"} | raw)
found.source = "ats"
return found
def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row:
[found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",))
return found
def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None:
store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2",
places=[{"text": "Aarhus", "remote": None}]))
raw = row(store, "1")
assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \
("SR", "Regular Full Time", "2")
assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \
("", "", "", "")
assert raw["prep_due"] == 1
assert store.role_places(raw["uid"])[0].place_id is None
done = prep.run(store, tables=TABLES)
assert (done.read, done.written, done.remapped) == (1, 1, True)
mapped = row(store, "1")
assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"],
mapped["remote"], mapped["country"], mapped["prep_due"]) == \
("senior", "senior", "full_time", "hybrid", "DK", 0)
assert store.role_places(mapped["uid"])[0].name == "Aarhus"
def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None:
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
store.upsert_role(item("2", seniority="sr"))
prep.run(store, tables=TABLES)
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
store.upsert_role(item("2", seniority="experienced"))
assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1)
# The place stays resolved when its text did not change.
assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus"
store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}]))
assert row(store, "1")["prep_due"] == 1
assert prep.run(store, tables=TABLES).read == 2
assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == ""
def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
store.upsert_role(item("1", seniority="sr"))
[read] = store.prep_rows("due")
store.upsert_role(item("1", seniority="experienced"))
store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)])
assert row(store, "1")["prep_due"] == 2
def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract(
store: Store) -> None:
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
uid = row(store, "1")["uid"]
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' "
"WHERE uid = ?", (uid,))
prep.run(store, tables=TABLES)
mapped = row(store, "1")
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
("senior", "lead", "remote")
# The host's own statement does overwrite.
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2"))
prep.run(store, tables=TABLES)
assert row(store, "1")["remote"] == "hybrid"
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(item("1", company="ORSTED A/S"))
assert row(store, "1")["company_key"] != "ørsted"
prep.run(store, tables=TABLES)
assert row(store, "1")["company_key"] == "ørsted"
def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None:
store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified",
country="Narnia"))
store.upsert_role(item("2", seniority="Grand Poobah", remote=False))
store.upsert_role(item("3", host="other", seniority="Grand Poobah"))
done = prep.run(store, tables=TABLES)
assert done.unmapped == 4
listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()}
assert listed == {("seniority", "acme", "grand poobah"): 2,
("seniority", "other", "grand poobah"): 1,
("remote", "acme", "unspecified"): 1,
("country", "acme", "narnia"): 1}
first = {w["word"]: w["first_seen"] for w in store.unmapped()}
# A word a table learns leaves the list at the next run; the rest keep
# the moment they were first met.
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}}
prep.run(store, tables=learned)
assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \
first
assert ("seniority", "acme", "grand poobah") not in \
{(w["field"], w["host"], w["word"]) for w in store.unmapped()}
assert row(store, "1")["seniority"] == "exec"
def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None:
store.upsert_role(item("1", seniority="Distinguished"))
store.upsert_role(item("2", seniority="Distinguished"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'")
assert prep.run(store, tables=TABLES).remapped
assert not prep.run(store, tables=TABLES).remapped
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"distinguished": "staff"}}}
done = prep.run(store, tables=learned)
assert (done.remapped, done.read) == (True, 1)
assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "")
# `--remap` maps the closed roles too.
assert prep.run(store, tables=learned, remap=True).read == 2
assert row(store, "2")["seniority"] == "staff"
def test_prep_folds_the_copies(store: Store) -> None:
for n in ("1", "2"):
store.upsert_role(item(n, place="Aarhus"))
assert prep.run(store, tables=TABLES).copies == 1
assert prep.run(store, tables=TABLES, dupes=False).copies is None
# --------------------------------------------------------------- migration
def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
path = tmp_path / "old.db"
store = Store(path)
store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True,
country="DK"))
store.upsert_role(item("2"))
store.state[prep.STATE] = "old"
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_prep_due")
for column in ("seniority_host", "employment_host", "remote_host", "country_host",
"employment", "prep_due"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
conn.execute("DROP TABLE unmapped")
conn.execute("PRAGMA user_version = 21")
migrated = Store(path)
assert migrated.migrations_applied == [22]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
assert row(migrated, "2")["remote_host"] == ""
assert prep.STATE not in migrated.state
# The first prep maps every open role from those words, the vocabulary
# values the old readers stored as themselves.
prep.run(migrated, tables={})
one = row(migrated, "1")
assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \
("senior", "full_time", "remote", "DK")
# ------------------------------------------------------------ CLI and API
def test_app_prep_runs_and_lists_the_unmapped_words(
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
path = tmp_path / "config.yaml"
path.write_text(f"database: {tmp_path / 'roledesk.db'}\n")
Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah"))
assert cli.main(["-c", str(path), "prep"]) == 0
assert json.loads(capsys.readouterr().out)["read"] == 1
assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0
assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"]
assert cli.main(["-c", str(path), "prep", "--remap"]) == 0
assert json.loads(capsys.readouterr().out)["remapped"] is True
def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None:
from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
app.state.store.upsert_role(item("1", employment="Zero hours"))
prep.run(app.state.store, tables=TABLES)
with TestClient(app) as client:
found = client.get("/api/prep/unmapped").json()
assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \
[("employment", "acme", "zero hours", 1)]
# ------------------------------------------------------------- the tables
def test_every_host_table_maps_into_its_vocabulary() -> None:
"""Passes with no table at all; R1's and R2's files are held here."""
for host, tables in hosts.load().items():
for field, table in tables.items():
if field == "match":
assert table["mode"] in hosts.MATCH_MODES, host
values = [value for _, value in table["rules"]]
field = "employment"
else:
assert all(key == key.strip().lower() for key in table), (host, field)
values = list(table.values())
assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field)
def test_a_match_table_must_say_how_it_matches(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n')
monkeypatch.setattr(hosts, "__path__", [str(tmp_path)])
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
with pytest.raises(ValueError, match=r"loose\.py has MATCH"):
hosts.load()
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n')
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
importlib.invalidate_caches()
assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]}
def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None:
assert _shared.EMPLOYMENTS == models.EMPLOYMENTS
+10 -8
View File
@@ -24,6 +24,7 @@ from app.models import Application # noqa: E402
from plugins import readers # noqa: E402
from plugins.readers import _runner # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app import prep # noqa: E402
from app.jobs import cascade, pull # noqa: E402
from app.storage import Store # noqa: E402
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 1, "closed": 1, "copies": 0}
"roles": 1, "closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -163,13 +164,14 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
"roles": 0, "closed": 0, "copies": 0}
"roles": 0, "closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
"""The settle's last step: a board's repeats of one job are one role,
and when the kept one closes the next copy is kept in its place."""
def test_the_prep_after_the_settle_rebuilds_the_copies(store: Store) -> None:
"""The prep that follows the settle folds: a board's repeats of one job
are one role, and when the kept one closes the next copy is kept in its
place."""
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
company="Acme", board="greenhouse")
for job_id in ("1", "2", "3"):
@@ -182,8 +184,8 @@ def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
class FakeTally:
results = ({"roles": 3, "run": "run-1"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result["copies"] == 2
pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert prep.run(store).copies == 2
[kept] = store.roles_page()
assert kept.copies == 2
@@ -191,7 +193,7 @@ def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
store.state["ats:ids:greenhouse:acme"] = json.dumps(
[f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone])
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert (result["closed"], result["copies"]) == (1, 1)
assert (result["closed"], prep.run(store).copies) == (1, 1)
[after] = store.roles_page()
assert after.uid != kept.uid and after.copies == 1
+19 -16
View File
@@ -24,6 +24,7 @@ from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
@@ -97,7 +98,7 @@ def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
title=title, body=body, published_at=published_at,
raw={"company": company, **raw})
item.source = "ats"
store.upsert_role(item)
upsert(store, item)
return item.uid
@@ -1023,7 +1024,7 @@ def test_the_shipped_starter_searches_validate() -> None:
# ---------------------------------------------------------------- cascade
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
@@ -1053,14 +1054,15 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
assert batch is not None
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
# Enrich found no board with role pages, so the embed and score behind it
# are passed over.
assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"]
# Enrich found no board with role pages, so the prep, embed and score
# behind it are passed over.
assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score",
"summarise"]
work = [j.kind for j in jobs if j.kind != FINAL]
# The `pull` job and the chain it asked for are one cascade, so the Jobs
# tab shows a scheduled pull as one run. Extract and summarise are off and
# find nothing; score runs twice, before and after extract.
assert work == ["pull", "pull_batch", "embed", "score", "score"]
assert work == ["pull", "pull_batch", "prep", "embed", "score", "score"]
assert all(j.state == "done" for j in jobs)
assert len(store.scores(agent.id)) == 1
# The board was stamped with the run the `pull` job claimed, and the
@@ -1109,11 +1111,12 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score",
"extract", "score", "summarise")
assert [cascade.follows("pull", at) for at in range(9)] == \
["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep",
"embed", "score", "extract", "score", "summarise")
assert [cascade.follows("pull", at) for at in range(11)] == \
["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score",
"summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == ""
assert cascade.follows("nope", 0) == ""
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
@@ -1126,10 +1129,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
made.append(t)
return t
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5},
context=Context())
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 8
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 10
+9 -4
View File
@@ -20,6 +20,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from app import prep # noqa: E402
from plugins import readers, search # noqa: E402
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402
@@ -28,6 +29,8 @@ from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
@@ -108,9 +111,9 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) ->
it.source = "ats"
return it
store.upsert_role(item("a", place="Copenhagen, Denmark"))
store.upsert_role(item("b", place="Remote", country="EU"))
store.upsert_role(item("c", place="Somewhere"))
upsert(store, item("a", place="Copenhagen, Denmark"))
upsert(store, item("b", place="Remote", country="EU"))
upsert(store, item("c", place="Somewhere"))
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
@@ -281,7 +284,9 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo
def _pull(store: Store) -> dict[str, Any]:
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
prep.run(store)
return closed
def test_a_complete_search_closes_what_it_stopped_listing(
+125 -124
View File
@@ -19,6 +19,7 @@ from app.models import (
)
from app.storage.store import pack
from app.storage import LATEST, Store
from prepped import upsert
class Clock:
@@ -69,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 21
assert LATEST == 22
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -186,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -236,9 +237,9 @@ def _downgrade_to_v4(path: Path) -> None:
def test_v6_moves_the_votes_and_affinities_to_the_default_agent(tmp_path: Path) -> None:
path = tmp_path / "v4.db"
store = Store(path)
store.upsert_role(posting("liked"))
store.upsert_role(posting("passed"))
store.upsert_role(posting("none"))
upsert(store, posting("liked"))
upsert(store, posting("passed"))
upsert(store, posting("none"))
store.add_company("Acme", key="acme")
store.add_company("Other", key="other")
# The disabled agent has the lower id; the enabled one is the default.
@@ -274,7 +275,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it(
tmp_path: Path) -> None:
path = tmp_path / "v4.db"
store = Store(path)
store.upsert_role(posting("liked"))
upsert(store, posting("liked"))
store.add_company("Acme", key="acme")
_downgrade_to_v4(path)
with sqlite3.connect(path) as conn:
@@ -292,7 +293,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it(
def test_v6_makes_no_agent_when_there_is_nothing_to_move(tmp_path: Path) -> None:
path = tmp_path / "v4.db"
store = Store(path)
store.upsert_role(posting("plain"))
upsert(store, posting("plain"))
store.add_company("Acme", key="acme")
_downgrade_to_v4(path)
@@ -322,8 +323,8 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
old_date = datetime(2020, 1, 1, tzinfo=UTC)
trusted = posting("1", board="greenhouse", published_at=old_date)
untrusted = posting("2", published_at=old_date)
store.upsert_role(trusted)
store.upsert_role(untrusted)
upsert(store, trusted)
upsert(store, untrusted)
before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid)
assert before_trusted is not None and before_untrusted is not None
@@ -354,7 +355,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
given = posting("3", place="Bogota, CO", country="US") # the board said so
extracted = posting("4", place="Bogota, CO")
for item in (wrong, several, given, extracted):
store.upsert_role(item)
upsert(store, item)
with sqlite3.connect(path) as conn:
_undo_v14(conn)
conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'")
@@ -375,7 +376,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: Path) -> None:
path = tmp_path / "v13.db"
store = Store(path)
store.upsert_role(posting("a"))
upsert(store, posting("a"))
uid = posting("a").uid
backend = store.save_agent("Backend")
blank = store.save_agent("Blank")
@@ -391,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -415,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -433,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
unknown = posting("unknown")
unknown.title = "Backend Engineer"
for item in (plain, extracted, kept, unknown):
store.upsert_role(item)
upsert(store, item)
with sqlite3.connect(path) as conn:
_undo_v17(conn)
conn.execute("UPDATE roles SET seniority = ''")
@@ -444,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -492,8 +493,8 @@ def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None:
def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage(
store: Store) -> None:
store.upsert_role(posting("a"))
store.upsert_role(posting("b"))
upsert(store, posting("a"))
upsert(store, posting("b"))
a, b = posting("a").uid, posting("b").uid
top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
blank = store.save_agent("Blank", limits=top,
@@ -585,11 +586,11 @@ def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
store: Store, clock: Clock) -> None:
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is True
assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is True
first = store.role(posting().uid)
assert first is not None
clock.tick()
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is False
assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is False
again = store.role(posting().uid)
assert again is not None
assert again.first_seen == first.first_seen == first.created_at
@@ -601,18 +602,18 @@ def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None:
listed = posting()
listed.body = ""
store.upsert_role(listed)
upsert(store, listed)
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
assert store.roles_to_embed("m", 10) == []
store.upsert_role(listed) # listed again, still without text
upsert(store, listed) # listed again, still without text
assert store.roles_to_embed("m", 10) == []
store.upsert_role(posting()) # the role's own page was read
upsert(store, posting()) # the role's own page was read
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(listed) # later pulls send no text
store.upsert_role(posting()) # or the same text again
upsert(store, listed) # later pulls send no text
upsert(store, posting()) # or the same text again
assert store.roles_to_embed("m", 10) == []
@@ -624,9 +625,9 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
listed.body = ""
bare = posting()
bare.body = ""
store.upsert_role(bare) # the title alone
upsert(store, bare) # the title alone
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(listed) # a teaser beats the title
upsert(store, listed) # a teaser beats the title
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and role.teaser == "Build the payments core."
@@ -634,21 +635,21 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
assert store.roles_page()[0].excerpt == "Build the payments core."
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(bare) # a pull without it keeps it
upsert(store, bare) # a pull without it keeps it
assert store.roles_to_embed("m", 10) == []
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
upsert(store, posting(teaser="Build the payments core.")) # the text beats it
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and document(role).endswith("We build things.")
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
upsert(store, posting(teaser="Another teaser.")) # a new teaser under the text
assert store.roles_to_embed("m", 10) == []
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
store.upsert_role(posting("1")) # "Senior Backend Engineer"
upsert(store, posting("1")) # "Senior Backend Engineer"
uid = posting("1").uid
def levels() -> tuple[str, str, str, str]:
@@ -658,12 +659,12 @@ def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> No
role.seniority_title)
assert levels() == ("senior", "", "", "senior")
store.upsert_role(posting("1", seniority="lead"))
upsert(store, posting("1", seniority="lead"))
assert levels() == ("lead", "", "lead", "senior")
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
upsert(store, posting("1", seniority="Teamleitung")) # not in the vocabulary
assert levels() == ("senior", "", "", "senior")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="lead"))
upsert(store, posting("1", seniority="lead"))
assert levels() == ("staff", "staff", "lead", "senior")
@@ -675,7 +676,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
stated = posting("stated", seniority="lead")
extracted = posting("extracted")
for item in (joined, detailed, stated, extracted):
store.upsert_role(item)
upsert(store, item)
with sqlite3.connect(path) as conn:
_undo_v19(conn)
# A role whose own page was read is pulled with an empty place in raw.
@@ -686,7 +687,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21]
assert migrated.migrations_applied == [19, 20, 21, 22]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -703,7 +704,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
upsert(store, posting(place="Berlin", remote=True, salary_min="80000",
salary_max=95000, currency="EUR", department="Eng"))
role = store.role(posting().uid)
assert role is not None
@@ -713,7 +714,7 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
assert role.description == "We build things."
assert role.original_url.endswith("/123")
# A later fetch saying less does not erase what was known.
store.upsert_role(posting())
upsert(store, posting())
role = store.role(posting().uid)
assert role is not None
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
@@ -725,16 +726,16 @@ def test_upsert_replaces_salary_text_including_with_empty(store: Store) -> None:
every pull, and a pull that states no pay at all clears it rather than
leaving a stale line behind — the opposite of the keep-filled rule
`test_upsert_unpacks_raw_and_keeps_filled_columns` checks just above."""
store.upsert_role(posting(salary_text="EUR 70,000 to 90,000 per year"))
upsert(store, posting(salary_text="EUR 70,000 to 90,000 per year"))
role = store.role(posting().uid)
assert role is not None and role.salary_text == "EUR 70,000 to 90,000 per year"
store.upsert_role(posting(salary_text="EUR 75,000 to 95,000 per year"))
upsert(store, posting(salary_text="EUR 75,000 to 95,000 per year"))
role = store.role(posting().uid)
assert role is not None and role.salary_text == "EUR 75,000 to 95,000 per year"
# The board stops stating pay: the stored line is cleared, not kept.
store.upsert_role(posting())
upsert(store, posting())
role = store.role(posting().uid)
assert role is not None and role.salary_text == ""
@@ -743,10 +744,10 @@ def test_salary_state_is_derived_from_the_columns_and_the_text(store: Store) ->
"""Not a stored column (docs/decisions.md): 'normalised' needs only
`salary_min` or `salary_max`, whatever `salary_text` says beside it;
'stated' is `salary_text` alone; neither is ''."""
store.upsert_role(posting("1", salary_text="EUR 70,000 to 90,000 per year",
upsert(store, posting("1", salary_text="EUR 70,000 to 90,000 per year",
salary_min=70000, salary_max=90000, currency="EUR"))
store.upsert_role(posting("2", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("3"))
upsert(store, posting("2", salary_text="USD 45 to 60 per hour"))
upsert(store, posting("3"))
by_id = {r.source_id: r for r in store.roles()}
assert by_id["1"].salary_state == "normalised"
assert by_id["2"].salary_state == "stated"
@@ -760,27 +761,27 @@ def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest(
that finally reports the true, earlier date must still win — an edit on
the board only ever moves published_at later, never earlier, so the
earliest posted_at ever computed is the one to keep."""
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20,
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 20,
tzinfo=UTC)))
first = store.role(posting().uid)
assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00"
clock.tick()
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1,
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 1, 1,
tzinfo=UTC)))
earlier = store.role(posting().uid)
assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00"
clock.tick()
# A later, LATER published_at (an edit) must not push posted_at forward.
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25,
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 25,
tzinfo=UTC)))
still_earlier = store.role(posting().uid)
assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00"
def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None:
store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
upsert(store, posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
role = store.role(posting().uid)
assert role is not None
assert role.posted_at == role.first_seen
@@ -791,7 +792,7 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No
only set it before extract has run, and once extract has answered — even
with a different function than the pull's guess — no later pull may put
the guess back. The no-flip property across pull, extract, pull."""
store.upsert_role(posting(function="engineering"))
upsert(store, posting(function="engineering"))
uid = posting().uid
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
@@ -801,28 +802,28 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No
# A later pull with the classifier's old (now wrong) guess must not flip
# it back — the "board wins" COALESCE that every other raw field gets
# would otherwise put "engineering" straight back on every pull.
store.upsert_role(posting(function="engineering"))
upsert(store, posting(function="engineering"))
assert store.role(uid).function == "sales" # type: ignore[union-attr]
def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None:
store.upsert_role(posting(function="engineering"))
upsert(store, posting(function="engineering"))
uid = posting().uid
store.set_extracted(uid, {}) # extract answered this role, but not function
assert store.role(uid).extracted_at is not None # type: ignore[union-attr]
# extracted_at alone freezes it, whatever a later pull's guess says.
store.upsert_role(posting(function="sales"))
upsert(store, posting(function="sales"))
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None:
store.upsert_role(posting(function="engineering"))
upsert(store, posting(function="engineering"))
role = store.role(posting().uid)
assert role is not None and role.function == "engineering"
# Before extract, a fresh guess still overwrites the old one, the same
# "board wins" behaviour as place, country and the rest.
store.upsert_role(posting(function="sales"))
upsert(store, posting(function="sales"))
role = store.role(posting().uid)
assert role is not None and role.function == "sales"
@@ -834,33 +835,33 @@ def _folded(store: Store, uid: str) -> str:
def test_place_fold_is_written_with_place(store: Store) -> None:
store.upsert_role(posting("1", place="København K"))
upsert(store, posting("1", place="København K"))
uid = posting("1").uid
assert _folded(store, uid) == "kobenhavn k"
store.upsert_role(posting("1")) # says less: both kept
upsert(store, posting("1")) # says less: both kept
assert (store.role(uid).place, _folded(store, uid)) == ( # type: ignore[union-attr]
"København K", "kobenhavn k")
store.upsert_role(posting("1", place="Kgs. Lyngby"))
upsert(store, posting("1", place="Kgs. Lyngby"))
assert _folded(store, uid) == "kgs lyngby"
store.upsert_role(posting("2"))
upsert(store, posting("2"))
store.set_extracted(posting("2").uid, {"place": "Malmö, Sweden"})
assert _folded(store, posting("2").uid) == "malmo sweden"
def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> None:
store.upsert_role(posting("1")) # "Senior Backend Engineer"
upsert(store, posting("1")) # "Senior Backend Engineer"
uid = posting("1").uid
assert store.role(uid).seniority == "senior" # type: ignore[union-attr]
retitled = posting("1")
retitled.title = "Backend Engineer"
store.upsert_role(retitled) # the title no longer says
upsert(store, retitled) # the title no longer says
assert store.role(uid).seniority == "" # type: ignore[union-attr]
store.upsert_role(posting("1"))
upsert(store, posting("1"))
store.set_extracted(uid, {"seniority": "lead"}) # extract overwrites the title's
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
store.upsert_role(posting("1")) # and a pull does not put it back
upsert(store, posting("1")) # and a pull does not put it back
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
store.set_extracted(uid, {})
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
@@ -868,14 +869,14 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
store: Store) -> None:
store.upsert_role(posting("1", seniority="junior"))
upsert(store, posting("1", seniority="junior"))
uid = posting("1").uid
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_title,
role.seniority_model) == ("junior", "junior", "senior", "")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="junior"))
upsert(store, posting("1", seniority="junior"))
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
@@ -884,18 +885,18 @@ def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
def test_company_falls_back_to_the_subscription(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
store.upsert_role(posting(), sub)
upsert(store, posting(), sub)
role = store.role(posting().uid)
assert role is not None
assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id)
def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin"))
upsert(store, posting("1", company="Acme", place="Berlin"))
clock.tick()
store.upsert_role(posting("2", company="Acme", place="Munich"))
upsert(store, posting("2", company="Acme", place="Munich"))
clock.tick()
store.upsert_role(posting("3", company="Other", place="Berlin"))
upsert(store, posting("3", company="Other", place="Berlin"))
assert [r.source_id for r in store.roles()] == ["3", "2", "1"]
assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"]
assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"]
@@ -905,9 +906,9 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None:
"""A role from a trusted board found today but posted months ago must not
outrank one the app has genuinely known about for longer."""
store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen
upsert(store, posting("recent")) # untrusted: posted_at=first_seen
clock.tick()
store.upsert_role(posting("old", board="greenhouse",
upsert(store, posting("old", board="greenhouse",
published_at=datetime(2020, 1, 1, tzinfo=UTC)))
assert [r.source_id for r in store.roles()] == ["recent", "old"]
assert [r.source_id for r in store.roles(days=30)] == ["recent"]
@@ -933,7 +934,7 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index(
can instead pick `idx_roles_closed` and sort every open role, which on
the live database (15,558 of 15,815 roles open) measured over 30x slower
than the plan this asserts (see docs/decisions.md)."""
store.upsert_role(posting("1"))
upsert(store, posting("1"))
sql, _ = store._roles_from(
company=None, place=None, open_only=True, q=None, function=None, remote=None,
days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any",
@@ -955,10 +956,10 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index(
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin",
upsert(store, posting("1", company="Acme", place="Berlin",
function="engineering", remote=True))
clock.tick(hours=48)
store.upsert_role(posting("2", company="Acme", place="Munich",
upsert(store, posting("2", company="Acme", place="Munich",
function="sales"))
assert [r.source_id for r in store.roles(q="senior engineer")] == ["2", "1"]
assert [r.source_id for r in store.roles(q="backend nonsense")] == []
@@ -977,12 +978,12 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No
"""`pay` filters in SQL, the same as the other filters `_roles_from`
builds, so it narrows what a `limit` is taken from rather than the page
already built (docs/decisions.md, "filtering before LIMIT")."""
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
upsert(store, posting("normalised", salary_min=70000, salary_max=90000,
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
clock.tick()
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
clock.tick()
store.upsert_role(posting("none"))
upsert(store, posting("none"))
assert {r.source_id for r in store.roles(pay="stated")} == {"normalised", "stated"}
assert [r.source_id for r in store.roles(pay="normalised")] == ["normalised"]
assert [r.source_id for r in store.roles(pay="stated_only")] == ["stated"]
@@ -993,9 +994,9 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No
def test_roles_filter_by_company_keys(store: Store) -> None:
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Acme Labs"))
store.upsert_role(posting("3", company="Other"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Acme Labs"))
upsert(store, posting("3", company="Other"))
assert {r.source_id for r in store.roles(company_keys=["acme"])} == {"1"}
assert {r.source_id for r in store.roles(company_keys=["acme", "other"])} == {"1", "3"}
assert store.count_roles(company_keys=["acme-labs"]) == 1
@@ -1008,7 +1009,7 @@ def test_roles_filter_by_place_tokens_on_whole_words(store: Store) -> None:
("offices", "Copenhagen; London; Warsaw", ""), ("aarhus", "Aarhus C", "DK"),
("malmo", "Malmö", ""), ("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
("berlin", "Berlin", "")):
store.upsert_role(posting(sid, place=place, country=country))
upsert(store, posting(sid, place=place, country=country))
def found(*tokens: str) -> set[str]:
return {r.source_id for r in store.roles(places=tokens)}
@@ -1030,7 +1031,7 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store
("lead", "Tech Lead"), ("none", "Engineer")):
item = posting(sid)
item.title = title
store.upsert_role(item)
upsert(store, item)
def found(*levels: str, unknown: bool = True) -> set[str]:
return {r.source_id for r in store.roles(seniority=levels, seniority_unknown=unknown)}
@@ -1046,9 +1047,9 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store
def test_roles_pay_unknown_lets_the_roles_with_no_pay_through(store: Store) -> None:
store.upsert_role(posting("normalised", salary_min=70000, currency="EUR"))
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("none"))
upsert(store, posting("normalised", salary_min=70000, currency="EUR"))
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
upsert(store, posting("none"))
assert {r.source_id for r in store.roles(pay="normalised", pay_unknown=True)} == \
{"normalised", "none"}
assert {r.source_id for r in store.roles(pay="stated", pay_unknown=True)} == \
@@ -1066,7 +1067,7 @@ def test_role_facets_count_each_facet_with_its_own_filter_left_out(store: Store)
("e", "Senior Engineer", {"company": "Other"})):
item = posting(sid, **raw)
item.title = title
store.upsert_role(item)
upsert(store, item)
facets = store.role_facets(seniority=["junior"], seniority_unknown=False,
pay="normalised")
@@ -1087,10 +1088,10 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store)
# One key, three spellings: the company row's name first, then the roles'
# by how many carry each.
store.add_company("Ørsted Group", key="ørsted")
store.upsert_role(posting("1", company="Ørsted A/S", place="Gentofte"))
store.upsert_role(posting("2", company="Ørsted", place="Gentofte"))
store.upsert_role(posting("3", company="Ørsted", place="Malmö"))
store.upsert_role(posting("4", company="Hidden", place="Malmö"))
upsert(store, posting("1", company="Ørsted A/S", place="Gentofte"))
upsert(store, posting("2", company="Ørsted", place="Gentofte"))
upsert(store, posting("3", company="Ørsted", place="Malmö"))
upsert(store, posting("4", company="Hidden", place="Malmö"))
agent = store.save_agent("A")
store.set_affinity(agent.id, "hidden", -1)
store.set_affinity(agent.id, "ørsted", 2)
@@ -1099,17 +1100,17 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store)
assert sorted(store.open_companies()) == [
("hidden", ["Hidden"], 1, 0), ("ørsted", spellings, 3, 0)]
assert store.open_companies(agent.id) == [("ørsted", spellings, 3, 2)]
assert sorted(store.open_places()) == [("Gentofte", "", 2), ("Malmö", "SE", 2)]
assert sorted(store.open_places()) == [("Gentofte", "DK", 2), ("Malmö", "SE", 2)]
def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None:
agent = store.save_agent("A")
store.add_company("Acme", key="acme")
store.set_affinity(agent.id, "acme", -1)
store.upsert_role(posting("other", company="Other"))
upsert(store, posting("other", company="Other"))
clock.tick()
for i in range(3):
store.upsert_role(posting(f"a{i}", company="Acme"))
upsert(store, posting(f"a{i}", company="Acme"))
clock.tick()
# Acme's roles are the three newest, so a plain limit=1 shows only Acme;
# hide_muted has to reach past all three to find "other" rather than
@@ -1128,7 +1129,7 @@ def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock)
def test_roles_page_by_offset_and_count_the_same_filter(
store: Store, clock: Clock) -> None:
for sid in ("1", "2", "3", "4"):
store.upsert_role(posting(sid, company="Acme" if sid != "4" else "Other"))
upsert(store, posting(sid, company="Acme" if sid != "4" else "Other"))
clock.tick()
assert [r.source_id for r in store.roles(limit=2, offset=1)] == ["3", "2"]
assert [r.source_id for r in store.roles(limit=2, offset=3)] == ["1"]
@@ -1142,7 +1143,7 @@ def test_roles_page_by_offset_and_count_the_same_filter(
def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> None:
for sid in ("up", "down", "fine", "none"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
role = store.set_vote(agent.id, uid["up"], 1)
@@ -1179,7 +1180,7 @@ def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> N
def test_votes_are_per_agent(store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
one = store.save_agent("One")
two = store.save_agent("Two")
@@ -1209,7 +1210,7 @@ def test_votes_are_per_agent(store: Store) -> None:
def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
store: Store, clock: Clock) -> None:
for sid in ("1", "2", "3"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
@@ -1226,7 +1227,7 @@ def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
def test_read_state_is_set_cleared_and_a_vote_marks_read(
store: Store, clock: Clock) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
role = store.role(uid)
assert role is not None and role.read_at is None
@@ -1252,8 +1253,8 @@ def test_read_state_is_set_cleared_and_a_vote_marks_read(
def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
upsert(store, posting("1"))
upsert(store, posting("2"))
uid = {r.source_id: r.uid for r in store.roles()}
store.set_read(uid["1"])
@@ -1263,7 +1264,7 @@ def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store)
def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
long_tag = "x" * 50
@@ -1283,9 +1284,9 @@ def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None:
def test_roles_filters_by_tags_any_and_all(store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
store.upsert_role(posting("3"))
upsert(store, posting("1"))
upsert(store, posting("2"))
upsert(store, posting("3"))
uid = {r.source_id: r.uid for r in store.roles()}
store.set_tags(uid["1"], ["python", "backend"])
store.set_tags(uid["2"], ["python", "frontend"])
@@ -1305,9 +1306,9 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) ->
store.add_company("Hidden", key="hidden")
agent = store.save_agent("A")
store.set_affinity(agent.id, "hidden", -1)
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Acme"))
store.upsert_role(posting("3", company="Hidden"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Acme"))
upsert(store, posting("3", company="Hidden"))
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
store.set_tags(uid["1"], ["python", "backend"])
store.set_tags(uid["2"], ["python"])
@@ -1320,7 +1321,7 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) ->
def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Store) -> None:
store.upsert_role(posting("a"))
upsert(store, posting("a"))
uid = store.roles()[0].uid
agent = store.save_agent("A")
store.delete_agent(agent.id)
@@ -1330,7 +1331,7 @@ def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Sto
def test_a_vote_moves_the_served_score_and_the_order(store: Store) -> None:
for sid in ("a", "b", "c"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
other = store.save_agent("B")
@@ -1354,13 +1355,13 @@ def test_mark_closed_closes_what_the_board_no_longer_lists(store: Store) -> None
sub = store.add_subscription("ats", "greenhouse:acme")
other = store.add_subscription("ats", "greenhouse:other")
for source_id in ("1", "2", "3"):
store.upsert_role(posting(source_id), sub)
store.upsert_role(posting("9"), other)
upsert(store, posting(source_id), sub)
upsert(store, posting("9"), other)
assert store.mark_closed("ats", ["1", "3"], sub.id) == 1
assert {r.source_id for r in store.roles()} == {"1", "3", "9"}
assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"}
# Listed again: open again.
store.upsert_role(posting("2"), sub)
upsert(store, posting("2"), sub)
assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"}
@@ -1368,7 +1369,7 @@ def test_an_item_without_a_source_is_refused(store: Store) -> None:
item = posting()
item.source = ""
with pytest.raises(ValueError, match="no source"):
store.upsert_role(item)
upsert(store, item)
# --------------------------------------------------------------- companies
@@ -1472,7 +1473,7 @@ def test_the_default_agent_is_the_lowest_enabled_id(store: Store) -> None:
def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None:
for sid, place, country in (("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
("kbh", "København K", ""), ("none", "", "")):
store.upsert_role(posting(sid, place=place, country=country))
upsert(store, posting(sid, place=place, country=country))
agent = store.save_agent("A")
uid = {r.source_id: r.uid for r in store.roles()}
@@ -1490,8 +1491,8 @@ def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None
def test_open_role_counts_and_company_boards(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme")
store.upsert_role(posting("1", company="Acme", place="Berlin"), sub)
store.upsert_role(posting("2", company="Acme", place="Munich"), sub)
upsert(store, posting("1", company="Acme", place="Berlin"), sub)
upsert(store, posting("2", company="Acme", place="Munich"), sub)
store.mark_closed("ats", ["1"], sub.id)
assert store.open_role_counts() == {"acme": 1}
assert store.company_boards() == {"acme": "greenhouse"}
@@ -1524,7 +1525,7 @@ def test_a_clash_on_replace_leaves_the_old_board_and_its_roles(store: Store) ->
store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme")
_, beta, _ = store.set_company_board("beta", "lever:beta", host="lever", name="Beta")
assert beta is not None
store.upsert_role(posting("1", company="Beta"), beta)
upsert(store, posting("1", company="Beta"), beta)
with pytest.raises(ValueError):
store.set_company_board("beta", "greenhouse:acme", host="greenhouse",
fields={"ats": "greenhouse"})
@@ -1539,8 +1540,8 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles(
name="Acme")
assert old is not None
store.update_subscription(old.id, fetch=False, target_id="gh-1")
store.upsert_role(posting("1", company="Acme"), old)
store.upsert_role(posting("2", company="Acme"), old)
upsert(store, posting("1", company="Acme"), old)
upsert(store, posting("2", company="Acme"), old)
company, new, closed = store.set_company_board("acme", "ashby:acme", host="ashby",
fields={"ats": "ashby"})
@@ -1550,7 +1551,7 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles(
"ashby:acme", "ashby", "", 1, 1)
assert company.ats == "ashby"
# The same board again closes nothing.
store.upsert_role(posting("3", company="Acme"), new)
upsert(store, posting("3", company="Acme"), new)
assert store.set_company_board("acme", "ashby:acme", host="ashby")[2] == 0
assert [r.source_id for r in store.roles()] == ["3"]
@@ -1560,7 +1561,7 @@ def test_detaching_a_board_hides_it_closes_its_roles_and_keeps_the_row(
_, sub, _ = store.set_company_board("acme", "greenhouse:acme", host="greenhouse",
name="Acme")
assert sub is not None
store.upsert_role(posting("1", company="Acme"), sub)
upsert(store, posting("1", company="Acme"), sub)
company, detached, closed = store.set_company_board("acme", "")
assert closed == 1 and company.key == "acme"
assert detached is not None
@@ -1627,7 +1628,7 @@ def test_events_are_append_only(store: Store) -> None:
def test_an_application_may_point_at_a_role(store: Store) -> None:
store.upsert_role(posting())
upsert(store, posting())
saved = store.save_application(Application(role_uid=posting().uid, status="applied"))
assert saved.role_uid == posting().uid
assert [e.status for e in store.application_events(saved.id)] == ["applied"]
@@ -1636,7 +1637,7 @@ def test_an_application_may_point_at_a_role(store: Store) -> None:
# -------------------------------------------------- summary and posting text
def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
store.upsert_role(posting())
upsert(store, posting())
role = store.set_article_text(posting().uid, "The whole posting.")
assert role is not None
assert role.article_text == "The whole posting." and role.article_at == store.now()
@@ -1644,7 +1645,7 @@ def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
def test_a_summary_is_written_once(store: Store) -> None:
store.upsert_role(posting())
upsert(store, posting())
uid = posting().uid
assert store.set_summary(uid, "First.") is True
assert store.set_summary(uid, "Second.") is False
@@ -1655,7 +1656,7 @@ def test_a_summary_is_written_once(store: Store) -> None:
def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer(
store: Store) -> None:
store.upsert_role(posting())
upsert(store, posting())
uid = posting().uid
assert store.set_summary(uid, "First.", ["Python", "backend"]) is True
role = store.role(uid)
@@ -1675,7 +1676,7 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
agent that hid it. An agent saved before v8 has `limits` of '{}', which
reads as the defaults."""
for sid in "abcd":
store.upsert_role(posting(sid))
upsert(store, posting(sid))
a, b, c, d = (posting(s).uid for s in "abcd")
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
+55 -54
View File
@@ -23,6 +23,7 @@ from app.web.api import _score_after_change
from app.web.pages import SECTIONS
from auth import AuthConfig
from desk.models import Item
from prepped import upsert
class Clock:
@@ -75,9 +76,9 @@ def client(app: FastAPI) -> Iterator[TestClient]:
def test_list_roles_filters_by_query_place_and_function(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering"))
store.upsert_role(posting("2", company="Acme", place="Munich", function="sales"))
store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering"))
upsert(store, posting("1", company="Acme", place="Berlin", function="engineering"))
upsert(store, posting("2", company="Acme", place="Munich", function="sales"))
upsert(store, posting("3", company="Other", place="Berlin", function="engineering"))
uid = {r.source_id: r.uid for r in store.roles()}
res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"})
@@ -100,7 +101,7 @@ def test_list_roles_filters_by_company_keys_places_and_seniority(
("berlin", "Other", "Berlin", "Senior Engineer")):
item = posting(sid, company=company, place=place)
item.title = title
store.upsert_role(item)
upsert(store, item)
uid = {r.source_id: r.uid for r in store.roles()}
def shown(**params: Any) -> set[str]:
@@ -133,7 +134,7 @@ def test_role_facets_count_each_facet_under_the_other_filters(
("c", "Engineer", {"salary_text": "EUR 50 per hour"})):
item = posting(sid, **raw)
item.title = title
store.upsert_role(item)
upsert(store, item)
res = client.get("/api/roles/facets",
params={"seniority": ["senior"], "seniority_unknown": "false",
@@ -152,7 +153,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden(
for sid, company in (("1", "Ørsted"), ("2", "Ørsted"), ("3", "Data Minds"),
("4", "Big Data Corp"), ("5", "Big Data Corp"), ("6", "Big Data Corp"),
("7", "Metadata Ltd"), ("8", "Hidden Data")):
store.upsert_role(posting(sid, company=company))
upsert(store, posting(sid, company=company))
agent = store.save_agent("A")
store.set_affinity(agent.id, "hidden-data", -1)
store.set_affinity(agent.id, "ørsted", 2)
@@ -177,7 +178,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden(
def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None:
for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"),
("4", "København K"), ("5", "Kolding")):
store.upsert_role(posting(sid, place=place))
upsert(store, posting(sid, place=place))
res = client.get("/api/suggest/places", params={"q": "ska"})
assert res.status_code == 200
@@ -200,10 +201,10 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store)
"""`salary_text` and `salary_state` (app/models.py) are on every role the
API serves, and `pay` (`Store.PAY`) filters the feed by it, in SQL before
`limit` like the other filters."""
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
upsert(store, posting("normalised", salary_min=70000, salary_max=90000,
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("none"))
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
upsert(store, posting("none"))
uid = {r.source_id: r.uid for r in store.roles()}
items = {r["uid"]: r for r in client.get("/api/roles").json()["items"]}
@@ -223,7 +224,7 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store)
def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
for i in range(3):
store.upsert_role(posting(str(i)))
upsert(store, posting(str(i)))
assert len(client.get("/api/roles", params={"limit": 1}).json()["items"]) == 1
assert client.get("/api/roles", params={"limit": 0}).status_code == 422
assert client.get("/api/roles", params={"limit": 1001}).status_code == 422
@@ -231,8 +232,8 @@ def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Store) -> None:
store.add_company("Acme", key="acme")
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Other"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Other"))
agent = store.save_agent("A")
other = store.save_agent("B")
uid = {r.source_id: r.uid for r in store.roles()}
@@ -265,10 +266,10 @@ def test_muted_is_excluded_before_the_limit_not_after(
store.clock = clock
store.add_company("Acme", key="acme")
store.set_affinity(store.save_agent("A").id, "acme", -1)
store.upsert_role(posting("other", company="Other"))
upsert(store, posting("other", company="Other"))
clock.tick()
for i in range(3):
store.upsert_role(posting(f"a{i}", company="Acme"))
upsert(store, posting(f"a{i}", company="Acme"))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
@@ -282,7 +283,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
clock = Clock()
store.clock = clock
for i in range(5):
store.upsert_role(posting(str(i)))
upsert(store, posting(str(i)))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
@@ -298,7 +299,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) -> None:
for sid in ("liked", "passed", "fine", "open"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
@@ -340,7 +341,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
def test_votes_are_per_agent_over_the_api(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
one = store.save_agent("One")
two = store.save_agent("Two")
@@ -364,7 +365,7 @@ def test_a_vote_or_an_affinity_needs_an_agent_to_belong_to(
client: TestClient, store: Store) -> None:
"""Nothing to vote as: a 422 with a sentence, not a vote that lands
nowhere. Reading is fine, everything is simply unjudged and neutral."""
store.upsert_role(posting("1"))
upsert(store, posting("1"))
store.add_company("Acme", key="acme")
uid = posting("1").uid
@@ -382,7 +383,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
clock = Clock()
store.clock = clock
for sid in ("1", "2", "3"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
@@ -403,7 +404,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
def test_a_vote_moves_a_role_in_the_ranking_at_once(
client: TestClient, store: Store) -> None:
for sid in ("a", "b", "c"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
label = {v: k for k, v in uid.items()}
agent = store.save_agent("A")
@@ -424,7 +425,7 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once(
def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
res = client.post(f"/api/roles/{uid}/read")
@@ -440,8 +441,8 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S
def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
upsert(store, posting("1"))
upsert(store, posting("2"))
ids = {r.source_id: r.uid for r in store.roles()}
uid = posting("1").uid
client.post(f"/api/roles/{uid}/read")
@@ -455,8 +456,8 @@ def test_list_roles_filters_by_include_read(client: TestClient, store: Store) ->
def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
upsert(store, posting("1"))
upsert(store, posting("2"))
uid = {r.source_id: r.uid for r in store.roles()}
store.set_tags(uid["1"], ["python", "backend"])
store.set_tags(uid["2"], ["python", "frontend"])
@@ -476,8 +477,8 @@ def test_filters_lists_tag_counts_and_respects_show_muted(
store.add_company("Hidden", key="hidden")
agent = store.save_agent("A") # the default agent, whose hidden companies count
store.set_affinity(agent.id, "hidden", -1)
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Hidden"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Hidden"))
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
store.set_tags(uid["1"], ["python"])
store.set_tags(uid["2"], ["python"])
@@ -491,7 +492,7 @@ def test_filters_lists_tag_counts_and_respects_show_muted(
def test_one_role_and_its_read_page(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin"))
upsert(store, posting("1", company="Acme", place="Berlin"))
uid = posting("1").uid
res = client.get(f"/api/roles/{uid}")
@@ -524,7 +525,7 @@ def test_companies_lists_board_and_open_role_count(
client: TestClient, store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme", ats="greenhouse", country="DK")
store.upsert_role(posting("1", company="Acme"), sub)
upsert(store, posting("1", company="Acme"), sub)
store.add_company("Other", key="other", domain="other.example") # no board found
store.add_company("Novo", key="novo", ats="taleo", # followed, not readable
careers_url="https://novo.example/careers")
@@ -561,7 +562,7 @@ def test_judging_a_company_known_only_from_a_search_creates_its_row(
for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")):
item = posting(f"n{i}", company=name)
item.source = "search"
store.upsert_role(item, search)
upsert(store, item, search)
key = "novo-nordisk"
assert store.company(key) is None
@@ -712,7 +713,7 @@ def test_the_watchlist_endpoints_404_for_an_unknown_company(client: TestClient)
def test_save_creates_an_application_with_a_saved_event(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin"))
upsert(store, posting("1", company="Acme", place="Berlin"))
uid = posting("1").uid
res = client.post("/api/applications", json={"role_uid": uid, "status": "saved"})
@@ -744,7 +745,7 @@ def test_save_an_unknown_role_is_refused(client: TestClient) -> None:
def test_set_status_validates_and_writes_an_event(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme"))
upsert(store, posting("1", company="Acme"))
uid = posting("1").uid
saved = client.post("/api/applications", json={"role_uid": uid}).json()
@@ -761,8 +762,8 @@ def test_set_status_validates_and_writes_an_event(
def test_unsave_drops_a_saved_application_and_refuses_one_that_moved_on(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Acme"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Acme"))
first = client.post("/api/applications", json={"role_uid": posting("1").uid}).json()
second = client.post("/api/applications", json={"role_uid": posting("2").uid}).json()
@@ -1069,7 +1070,7 @@ def test_a_duplicate_or_empty_prompt_name_is_a_422_sentence(
def test_the_role_endpoint_serves_the_agents_score_and_its_prompt(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
agent = store.save_agent("A", prompts=[])
store.replace_scores(agent.id, [(uid, 0.8, "", "Forward deployed")])
@@ -1088,7 +1089,7 @@ def test_the_role_endpoint_serves_the_agents_score_and_its_prompt(
def test_deleting_an_agent_takes_its_votes_and_affinities_with_it(
client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
store.add_company("Acme", key="acme")
uid = posting("1").uid
agent = client.post("/api/agents", json={"name": "A"}).json()
@@ -1127,7 +1128,7 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No
def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
client: TestClient, store: Store) -> None:
for sid in ("low", "high", "gated"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uids = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
store.replace_scores(agent.id, [(uids["low"], 0.2, ""), (uids["high"], 0.8, ""),
@@ -1152,7 +1153,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid",
function="engineering")
item.body = long_description
store.upsert_role(item)
upsert(store, item)
uid = item.uid
row = client.get("/api/roles").json()["items"][0]
@@ -1184,7 +1185,7 @@ def test_list_roles_page_stays_small_even_with_long_postings(
for n in range(60):
item = posting(str(n), company="Acme")
item.body = big
store.upsert_role(item)
upsert(store, item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?",
(big, json.dumps({"blob": big}), big))
@@ -1197,7 +1198,7 @@ def test_list_roles_page_stays_small_even_with_long_postings(
def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
store.set_summary(uid, "A short, hand-written summary.")
row = client.get("/api/roles").json()["items"][0]
@@ -1221,9 +1222,9 @@ def test_sources_lists_searches_and_boards_with_their_countries(
client: TestClient, store: Store) -> None:
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
store.upsert_role(posting("1", company="Acme", place="Copenhagen"), board)
store.upsert_role(posting("2", company="Acme", place="Boston, MA"), board)
store.upsert_role(posting("3", company="Acme", place="Aarhus"), board)
upsert(store, posting("1", company="Acme", place="Copenhagen"), board)
upsert(store, posting("2", company="Acme", place="Boston, MA"), board)
upsert(store, posting("3", company="Acme", place="Aarhus"), board)
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend", "area": 1})
@@ -1244,7 +1245,7 @@ def test_switching_a_board_off_closes_its_roles_and_on_queues_a_pull(
client: TestClient, store: Store, queue: Any) -> None:
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
store.upsert_role(posting("1", company="Acme"), board)
upsert(store, posting("1", company="Acme"), board)
off = client.post(f"/api/sources/{board.id}/switch", json={"on": False}).json()
assert off["closed"] == 1 and off["source"]["on"] is False
@@ -1345,7 +1346,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
body = res.json()
assert (body["found"], body["closed"], body["source"]["target"]) == (0, 0, "greenhouse:acme")
sub_id = body["source"]["id"]
store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id))
upsert(store, posting("1", company="Acme"), store.subscription(sub_id))
[row] = client.get("/api/companies").json()
assert (row["board"], row["subscription_id"], row["ats"], row["ats_slug"]) == (
"greenhouse", sub_id, "greenhouse", "acme")
@@ -1418,7 +1419,7 @@ def test_a_company_from_a_search_only_is_no_longer_one_once_it_has_a_reason(
search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex")
item = posting("1", company="Novo")
item.source = "search"
store.upsert_role(item, search)
upsert(store, item, search)
client.post("/api/companies/novo/affinity", json={"affinity": 1})
[row] = client.get("/api/companies").json()
assert (row["from_search"], row["by_hand"]) == (True, False)
@@ -1514,14 +1515,14 @@ def test_a_company_is_deleted_unless_it_is_read(
).json()["source"]["id"]
res = client.delete("/api/companies/acme")
assert res.status_code == 409 and "greenhouse:acme" in res.json()["detail"]
store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id))
upsert(store, posting("1", company="Acme"), store.subscription(sub_id))
client.delete("/api/companies/acme/board")
assert client.delete("/api/companies/acme").status_code == 204
# Open roles hold a company even where no board is on: they would be
# orphaned from the list.
store.add_company("Beta", key="beta")
store.upsert_role(posting("2", company="Beta"))
upsert(store, posting("2", company="Beta"))
res = client.delete("/api/companies/beta")
assert res.status_code == 409 and "open roles" in res.json()["detail"]
assert store.company("beta") is not None
@@ -1562,7 +1563,7 @@ def test_the_posting_text_is_fetched_once(
from desk.tools import scrape
from desk.tools.article import ArticleContent
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
asked: list[str] = []
@@ -1608,8 +1609,8 @@ def test_summarise_queues_one_role_and_the_job_writes_it(
monkeypatch.setitem(JOBS, "summarise_batch",
replace(JOBS["summarise_batch"], isolate=False, timeout=0.0))
client = TestClient(app)
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
upsert(store, posting("1"))
upsert(store, posting("2"))
uid, bare = posting("1").uid, posting("2").uid
with store.transaction() as conn:
conn.execute("UPDATE roles SET description = '' WHERE uid = ?", (bare,))
@@ -1656,7 +1657,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
board.body = "The whole posting."
search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex")
for item in (board, search, posting("alone", company="Other")):
store.upsert_role(item)
upsert(store, item)
store.rebuild_dupes()
page = client.get("/api/roles").json()
@@ -1675,7 +1676,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
def test_suggest_companies_matches_through_an_alias(client: TestClient, store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(posting("1", company="Orsted"))
upsert(store, posting("1", company="Orsted"))
store.add_alias("ørsted", "Dong Energy")
for q in ("orsted", "ørsted", "dong"):