Prep stage: host words stored raw, mapped centrally in app/prep after every pull

upsert_role stores the host's own words (roles.*_host), the place texts
unresolved when they change, and the company key unresolved on insert, and
marks the role prep_due. app.prep maps seniority, employment, remote and
country through the host tables in app/prep/hosts, resolves places and
company aliases, lists unknown words in `unmapped`, remaps every open role
when the tables' fingerprint changes, and runs rebuild_dupes, which moves
out of the pull's settle. Migration 22 adds the columns and the table and
fills the host words from raw. Cascade: pull, prep, embed, score, enrich,
prep, embed, score, extract, score, summarise. `app prep [unmapped]
[--remap]`, /api/prep/unmapped and a fold on the Sources page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:19:22 +02:00
1 parent a66aac95cf
commit d04ceae7e0
31 files changed
+1468 -330

No files matched your search

+125 -124
View File
@@ -19,6 +19,7 @@ from app.models import (
)
from app.storage.store import pack
from app.storage import LATEST, Store
from prepped import upsert
class Clock:
@@ -69,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 21
assert LATEST == 22
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -186,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -236,9 +237,9 @@ def _downgrade_to_v4(path: Path) -> None:
def test_v6_moves_the_votes_and_affinities_to_the_default_agent(tmp_path: Path) -> None:
path = tmp_path / "v4.db"
store = Store(path)
store.upsert_role(posting("liked"))
store.upsert_role(posting("passed"))
store.upsert_role(posting("none"))
upsert(store, posting("liked"))
upsert(store, posting("passed"))
upsert(store, posting("none"))
store.add_company("Acme", key="acme")
store.add_company("Other", key="other")
# The disabled agent has the lower id; the enabled one is the default.
@@ -274,7 +275,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it(
tmp_path: Path) -> None:
path = tmp_path / "v4.db"
store = Store(path)
store.upsert_role(posting("liked"))
upsert(store, posting("liked"))
store.add_company("Acme", key="acme")
_downgrade_to_v4(path)
with sqlite3.connect(path) as conn:
@@ -292,7 +293,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it(
def test_v6_makes_no_agent_when_there_is_nothing_to_move(tmp_path: Path) -> None:
path = tmp_path / "v4.db"
store = Store(path)
store.upsert_role(posting("plain"))
upsert(store, posting("plain"))
store.add_company("Acme", key="acme")
_downgrade_to_v4(path)
@@ -322,8 +323,8 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
old_date = datetime(2020, 1, 1, tzinfo=UTC)
trusted = posting("1", board="greenhouse", published_at=old_date)
untrusted = posting("2", published_at=old_date)
store.upsert_role(trusted)
store.upsert_role(untrusted)
upsert(store, trusted)
upsert(store, untrusted)
before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid)
assert before_trusted is not None and before_untrusted is not None
@@ -354,7 +355,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
given = posting("3", place="Bogota, CO", country="US") # the board said so
extracted = posting("4", place="Bogota, CO")
for item in (wrong, several, given, extracted):
store.upsert_role(item)
upsert(store, item)
with sqlite3.connect(path) as conn:
_undo_v14(conn)
conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'")
@@ -375,7 +376,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: Path) -> None:
path = tmp_path / "v13.db"
store = Store(path)
store.upsert_role(posting("a"))
upsert(store, posting("a"))
uid = posting("a").uid
backend = store.save_agent("Backend")
blank = store.save_agent("Blank")
@@ -391,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -415,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -433,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
unknown = posting("unknown")
unknown.title = "Backend Engineer"
for item in (plain, extracted, kept, unknown):
store.upsert_role(item)
upsert(store, item)
with sqlite3.connect(path) as conn:
_undo_v17(conn)
conn.execute("UPDATE roles SET seniority = ''")
@@ -444,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -492,8 +493,8 @@ def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None:
def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage(
store: Store) -> None:
store.upsert_role(posting("a"))
store.upsert_role(posting("b"))
upsert(store, posting("a"))
upsert(store, posting("b"))
a, b = posting("a").uid, posting("b").uid
top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
blank = store.save_agent("Blank", limits=top,
@@ -585,11 +586,11 @@ def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
store: Store, clock: Clock) -> None:
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is True
assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is True
first = store.role(posting().uid)
assert first is not None
clock.tick()
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is False
assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is False
again = store.role(posting().uid)
assert again is not None
assert again.first_seen == first.first_seen == first.created_at
@@ -601,18 +602,18 @@ def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None:
listed = posting()
listed.body = ""
store.upsert_role(listed)
upsert(store, listed)
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
assert store.roles_to_embed("m", 10) == []
store.upsert_role(listed) # listed again, still without text
upsert(store, listed) # listed again, still without text
assert store.roles_to_embed("m", 10) == []
store.upsert_role(posting()) # the role's own page was read
upsert(store, posting()) # the role's own page was read
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(listed) # later pulls send no text
store.upsert_role(posting()) # or the same text again
upsert(store, listed) # later pulls send no text
upsert(store, posting()) # or the same text again
assert store.roles_to_embed("m", 10) == []
@@ -624,9 +625,9 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
listed.body = ""
bare = posting()
bare.body = ""
store.upsert_role(bare) # the title alone
upsert(store, bare) # the title alone
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(listed) # a teaser beats the title
upsert(store, listed) # a teaser beats the title
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and role.teaser == "Build the payments core."
@@ -634,21 +635,21 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
assert store.roles_page()[0].excerpt == "Build the payments core."
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(bare) # a pull without it keeps it
upsert(store, bare) # a pull without it keeps it
assert store.roles_to_embed("m", 10) == []
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
upsert(store, posting(teaser="Build the payments core.")) # the text beats it
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
role = store.role(listed.uid)
assert role is not None and document(role).endswith("We build things.")
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
upsert(store, posting(teaser="Another teaser.")) # a new teaser under the text
assert store.roles_to_embed("m", 10) == []
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
store.upsert_role(posting("1")) # "Senior Backend Engineer"
upsert(store, posting("1")) # "Senior Backend Engineer"
uid = posting("1").uid
def levels() -> tuple[str, str, str, str]:
@@ -658,12 +659,12 @@ def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> No
role.seniority_title)
assert levels() == ("senior", "", "", "senior")
store.upsert_role(posting("1", seniority="lead"))
upsert(store, posting("1", seniority="lead"))
assert levels() == ("lead", "", "lead", "senior")
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
upsert(store, posting("1", seniority="Teamleitung")) # not in the vocabulary
assert levels() == ("senior", "", "", "senior")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="lead"))
upsert(store, posting("1", seniority="lead"))
assert levels() == ("staff", "staff", "lead", "senior")
@@ -675,7 +676,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
stated = posting("stated", seniority="lead")
extracted = posting("extracted")
for item in (joined, detailed, stated, extracted):
store.upsert_role(item)
upsert(store, item)
with sqlite3.connect(path) as conn:
_undo_v19(conn)
# A role whose own page was read is pulled with an empty place in raw.
@@ -686,7 +687,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21]
assert migrated.migrations_applied == [19, 20, 21, 22]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -703,7 +704,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
upsert(store, posting(place="Berlin", remote=True, salary_min="80000",
salary_max=95000, currency="EUR", department="Eng"))
role = store.role(posting().uid)
assert role is not None
@@ -713,7 +714,7 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
assert role.description == "We build things."
assert role.original_url.endswith("/123")
# A later fetch saying less does not erase what was known.
store.upsert_role(posting())
upsert(store, posting())
role = store.role(posting().uid)
assert role is not None
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
@@ -725,16 +726,16 @@ def test_upsert_replaces_salary_text_including_with_empty(store: Store) -> None:
every pull, and a pull that states no pay at all clears it rather than
leaving a stale line behind — the opposite of the keep-filled rule
`test_upsert_unpacks_raw_and_keeps_filled_columns` checks just above."""
store.upsert_role(posting(salary_text="EUR 70,000 to 90,000 per year"))
upsert(store, posting(salary_text="EUR 70,000 to 90,000 per year"))
role = store.role(posting().uid)
assert role is not None and role.salary_text == "EUR 70,000 to 90,000 per year"
store.upsert_role(posting(salary_text="EUR 75,000 to 95,000 per year"))
upsert(store, posting(salary_text="EUR 75,000 to 95,000 per year"))
role = store.role(posting().uid)
assert role is not None and role.salary_text == "EUR 75,000 to 95,000 per year"
# The board stops stating pay: the stored line is cleared, not kept.
store.upsert_role(posting())
upsert(store, posting())
role = store.role(posting().uid)
assert role is not None and role.salary_text == ""
@@ -743,10 +744,10 @@ def test_salary_state_is_derived_from_the_columns_and_the_text(store: Store) ->
"""Not a stored column (docs/decisions.md): 'normalised' needs only
`salary_min` or `salary_max`, whatever `salary_text` says beside it;
'stated' is `salary_text` alone; neither is ''."""
store.upsert_role(posting("1", salary_text="EUR 70,000 to 90,000 per year",
upsert(store, posting("1", salary_text="EUR 70,000 to 90,000 per year",
salary_min=70000, salary_max=90000, currency="EUR"))
store.upsert_role(posting("2", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("3"))
upsert(store, posting("2", salary_text="USD 45 to 60 per hour"))
upsert(store, posting("3"))
by_id = {r.source_id: r for r in store.roles()}
assert by_id["1"].salary_state == "normalised"
assert by_id["2"].salary_state == "stated"
@@ -760,27 +761,27 @@ def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest(
that finally reports the true, earlier date must still win — an edit on
the board only ever moves published_at later, never earlier, so the
earliest posted_at ever computed is the one to keep."""
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20,
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 20,
tzinfo=UTC)))
first = store.role(posting().uid)
assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00"
clock.tick()
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1,
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 1, 1,
tzinfo=UTC)))
earlier = store.role(posting().uid)
assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00"
clock.tick()
# A later, LATER published_at (an edit) must not push posted_at forward.
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25,
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 25,
tzinfo=UTC)))
still_earlier = store.role(posting().uid)
assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00"
def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None:
store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
upsert(store, posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
role = store.role(posting().uid)
assert role is not None
assert role.posted_at == role.first_seen
@@ -791,7 +792,7 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No
only set it before extract has run, and once extract has answered — even
with a different function than the pull's guess — no later pull may put
the guess back. The no-flip property across pull, extract, pull."""
store.upsert_role(posting(function="engineering"))
upsert(store, posting(function="engineering"))
uid = posting().uid
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
@@ -801,28 +802,28 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No
# A later pull with the classifier's old (now wrong) guess must not flip
# it back — the "board wins" COALESCE that every other raw field gets
# would otherwise put "engineering" straight back on every pull.
store.upsert_role(posting(function="engineering"))
upsert(store, posting(function="engineering"))
assert store.role(uid).function == "sales" # type: ignore[union-attr]
def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None:
store.upsert_role(posting(function="engineering"))
upsert(store, posting(function="engineering"))
uid = posting().uid
store.set_extracted(uid, {}) # extract answered this role, but not function
assert store.role(uid).extracted_at is not None # type: ignore[union-attr]
# extracted_at alone freezes it, whatever a later pull's guess says.
store.upsert_role(posting(function="sales"))
upsert(store, posting(function="sales"))
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None:
store.upsert_role(posting(function="engineering"))
upsert(store, posting(function="engineering"))
role = store.role(posting().uid)
assert role is not None and role.function == "engineering"
# Before extract, a fresh guess still overwrites the old one, the same
# "board wins" behaviour as place, country and the rest.
store.upsert_role(posting(function="sales"))
upsert(store, posting(function="sales"))
role = store.role(posting().uid)
assert role is not None and role.function == "sales"
@@ -834,33 +835,33 @@ def _folded(store: Store, uid: str) -> str:
def test_place_fold_is_written_with_place(store: Store) -> None:
store.upsert_role(posting("1", place="København K"))
upsert(store, posting("1", place="København K"))
uid = posting("1").uid
assert _folded(store, uid) == "kobenhavn k"
store.upsert_role(posting("1")) # says less: both kept
upsert(store, posting("1")) # says less: both kept
assert (store.role(uid).place, _folded(store, uid)) == ( # type: ignore[union-attr]
"København K", "kobenhavn k")
store.upsert_role(posting("1", place="Kgs. Lyngby"))
upsert(store, posting("1", place="Kgs. Lyngby"))
assert _folded(store, uid) == "kgs lyngby"
store.upsert_role(posting("2"))
upsert(store, posting("2"))
store.set_extracted(posting("2").uid, {"place": "Malmö, Sweden"})
assert _folded(store, posting("2").uid) == "malmo sweden"
def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> None:
store.upsert_role(posting("1")) # "Senior Backend Engineer"
upsert(store, posting("1")) # "Senior Backend Engineer"
uid = posting("1").uid
assert store.role(uid).seniority == "senior" # type: ignore[union-attr]
retitled = posting("1")
retitled.title = "Backend Engineer"
store.upsert_role(retitled) # the title no longer says
upsert(store, retitled) # the title no longer says
assert store.role(uid).seniority == "" # type: ignore[union-attr]
store.upsert_role(posting("1"))
upsert(store, posting("1"))
store.set_extracted(uid, {"seniority": "lead"}) # extract overwrites the title's
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
store.upsert_role(posting("1")) # and a pull does not put it back
upsert(store, posting("1")) # and a pull does not put it back
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
store.set_extracted(uid, {})
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
@@ -868,14 +869,14 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
store: Store) -> None:
store.upsert_role(posting("1", seniority="junior"))
upsert(store, posting("1", seniority="junior"))
uid = posting("1").uid
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_title,
role.seniority_model) == ("junior", "junior", "senior", "")
store.set_extracted(uid, {"seniority": "staff"})
store.upsert_role(posting("1", seniority="junior"))
upsert(store, posting("1", seniority="junior"))
role = store.role(uid)
assert role is not None
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
@@ -884,18 +885,18 @@ def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
def test_company_falls_back_to_the_subscription(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
store.upsert_role(posting(), sub)
upsert(store, posting(), sub)
role = store.role(posting().uid)
assert role is not None
assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id)
def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin"))
upsert(store, posting("1", company="Acme", place="Berlin"))
clock.tick()
store.upsert_role(posting("2", company="Acme", place="Munich"))
upsert(store, posting("2", company="Acme", place="Munich"))
clock.tick()
store.upsert_role(posting("3", company="Other", place="Berlin"))
upsert(store, posting("3", company="Other", place="Berlin"))
assert [r.source_id for r in store.roles()] == ["3", "2", "1"]
assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"]
assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"]
@@ -905,9 +906,9 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None:
"""A role from a trusted board found today but posted months ago must not
outrank one the app has genuinely known about for longer."""
store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen
upsert(store, posting("recent")) # untrusted: posted_at=first_seen
clock.tick()
store.upsert_role(posting("old", board="greenhouse",
upsert(store, posting("old", board="greenhouse",
published_at=datetime(2020, 1, 1, tzinfo=UTC)))
assert [r.source_id for r in store.roles()] == ["recent", "old"]
assert [r.source_id for r in store.roles(days=30)] == ["recent"]
@@ -933,7 +934,7 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index(
can instead pick `idx_roles_closed` and sort every open role, which on
the live database (15,558 of 15,815 roles open) measured over 30x slower
than the plan this asserts (see docs/decisions.md)."""
store.upsert_role(posting("1"))
upsert(store, posting("1"))
sql, _ = store._roles_from(
company=None, place=None, open_only=True, q=None, function=None, remote=None,
days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any",
@@ -955,10 +956,10 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index(
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
store.upsert_role(posting("1", company="Acme", place="Berlin",
upsert(store, posting("1", company="Acme", place="Berlin",
function="engineering", remote=True))
clock.tick(hours=48)
store.upsert_role(posting("2", company="Acme", place="Munich",
upsert(store, posting("2", company="Acme", place="Munich",
function="sales"))
assert [r.source_id for r in store.roles(q="senior engineer")] == ["2", "1"]
assert [r.source_id for r in store.roles(q="backend nonsense")] == []
@@ -977,12 +978,12 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No
"""`pay` filters in SQL, the same as the other filters `_roles_from`
builds, so it narrows what a `limit` is taken from rather than the page
already built (docs/decisions.md, "filtering before LIMIT")."""
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
upsert(store, posting("normalised", salary_min=70000, salary_max=90000,
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
clock.tick()
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
clock.tick()
store.upsert_role(posting("none"))
upsert(store, posting("none"))
assert {r.source_id for r in store.roles(pay="stated")} == {"normalised", "stated"}
assert [r.source_id for r in store.roles(pay="normalised")] == ["normalised"]
assert [r.source_id for r in store.roles(pay="stated_only")] == ["stated"]
@@ -993,9 +994,9 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No
def test_roles_filter_by_company_keys(store: Store) -> None:
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Acme Labs"))
store.upsert_role(posting("3", company="Other"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Acme Labs"))
upsert(store, posting("3", company="Other"))
assert {r.source_id for r in store.roles(company_keys=["acme"])} == {"1"}
assert {r.source_id for r in store.roles(company_keys=["acme", "other"])} == {"1", "3"}
assert store.count_roles(company_keys=["acme-labs"]) == 1
@@ -1008,7 +1009,7 @@ def test_roles_filter_by_place_tokens_on_whole_words(store: Store) -> None:
("offices", "Copenhagen; London; Warsaw", ""), ("aarhus", "Aarhus C", "DK"),
("malmo", "Malmö", ""), ("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
("berlin", "Berlin", "")):
store.upsert_role(posting(sid, place=place, country=country))
upsert(store, posting(sid, place=place, country=country))
def found(*tokens: str) -> set[str]:
return {r.source_id for r in store.roles(places=tokens)}
@@ -1030,7 +1031,7 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store
("lead", "Tech Lead"), ("none", "Engineer")):
item = posting(sid)
item.title = title
store.upsert_role(item)
upsert(store, item)
def found(*levels: str, unknown: bool = True) -> set[str]:
return {r.source_id for r in store.roles(seniority=levels, seniority_unknown=unknown)}
@@ -1046,9 +1047,9 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store
def test_roles_pay_unknown_lets_the_roles_with_no_pay_through(store: Store) -> None:
store.upsert_role(posting("normalised", salary_min=70000, currency="EUR"))
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("none"))
upsert(store, posting("normalised", salary_min=70000, currency="EUR"))
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
upsert(store, posting("none"))
assert {r.source_id for r in store.roles(pay="normalised", pay_unknown=True)} == \
{"normalised", "none"}
assert {r.source_id for r in store.roles(pay="stated", pay_unknown=True)} == \
@@ -1066,7 +1067,7 @@ def test_role_facets_count_each_facet_with_its_own_filter_left_out(store: Store)
("e", "Senior Engineer", {"company": "Other"})):
item = posting(sid, **raw)
item.title = title
store.upsert_role(item)
upsert(store, item)
facets = store.role_facets(seniority=["junior"], seniority_unknown=False,
pay="normalised")
@@ -1087,10 +1088,10 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store)
# One key, three spellings: the company row's name first, then the roles'
# by how many carry each.
store.add_company("Ørsted Group", key="ørsted")
store.upsert_role(posting("1", company="Ørsted A/S", place="Gentofte"))
store.upsert_role(posting("2", company="Ørsted", place="Gentofte"))
store.upsert_role(posting("3", company="Ørsted", place="Malmö"))
store.upsert_role(posting("4", company="Hidden", place="Malmö"))
upsert(store, posting("1", company="Ørsted A/S", place="Gentofte"))
upsert(store, posting("2", company="Ørsted", place="Gentofte"))
upsert(store, posting("3", company="Ørsted", place="Malmö"))
upsert(store, posting("4", company="Hidden", place="Malmö"))
agent = store.save_agent("A")
store.set_affinity(agent.id, "hidden", -1)
store.set_affinity(agent.id, "ørsted", 2)
@@ -1099,17 +1100,17 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store)
assert sorted(store.open_companies()) == [
("hidden", ["Hidden"], 1, 0), ("ørsted", spellings, 3, 0)]
assert store.open_companies(agent.id) == [("ørsted", spellings, 3, 2)]
assert sorted(store.open_places()) == [("Gentofte", "", 2), ("Malmö", "SE", 2)]
assert sorted(store.open_places()) == [("Gentofte", "DK", 2), ("Malmö", "SE", 2)]
def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None:
agent = store.save_agent("A")
store.add_company("Acme", key="acme")
store.set_affinity(agent.id, "acme", -1)
store.upsert_role(posting("other", company="Other"))
upsert(store, posting("other", company="Other"))
clock.tick()
for i in range(3):
store.upsert_role(posting(f"a{i}", company="Acme"))
upsert(store, posting(f"a{i}", company="Acme"))
clock.tick()
# Acme's roles are the three newest, so a plain limit=1 shows only Acme;
# hide_muted has to reach past all three to find "other" rather than
@@ -1128,7 +1129,7 @@ def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock)
def test_roles_page_by_offset_and_count_the_same_filter(
store: Store, clock: Clock) -> None:
for sid in ("1", "2", "3", "4"):
store.upsert_role(posting(sid, company="Acme" if sid != "4" else "Other"))
upsert(store, posting(sid, company="Acme" if sid != "4" else "Other"))
clock.tick()
assert [r.source_id for r in store.roles(limit=2, offset=1)] == ["3", "2"]
assert [r.source_id for r in store.roles(limit=2, offset=3)] == ["1"]
@@ -1142,7 +1143,7 @@ def test_roles_page_by_offset_and_count_the_same_filter(
def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> None:
for sid in ("up", "down", "fine", "none"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
role = store.set_vote(agent.id, uid["up"], 1)
@@ -1179,7 +1180,7 @@ def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> N
def test_votes_are_per_agent(store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
one = store.save_agent("One")
two = store.save_agent("Two")
@@ -1209,7 +1210,7 @@ def test_votes_are_per_agent(store: Store) -> None:
def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
store: Store, clock: Clock) -> None:
for sid in ("1", "2", "3"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
@@ -1226,7 +1227,7 @@ def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
def test_read_state_is_set_cleared_and_a_vote_marks_read(
store: Store, clock: Clock) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
role = store.role(uid)
assert role is not None and role.read_at is None
@@ -1252,8 +1253,8 @@ def test_read_state_is_set_cleared_and_a_vote_marks_read(
def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
upsert(store, posting("1"))
upsert(store, posting("2"))
uid = {r.source_id: r.uid for r in store.roles()}
store.set_read(uid["1"])
@@ -1263,7 +1264,7 @@ def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store)
def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
long_tag = "x" * 50
@@ -1283,9 +1284,9 @@ def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None:
def test_roles_filters_by_tags_any_and_all(store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
store.upsert_role(posting("3"))
upsert(store, posting("1"))
upsert(store, posting("2"))
upsert(store, posting("3"))
uid = {r.source_id: r.uid for r in store.roles()}
store.set_tags(uid["1"], ["python", "backend"])
store.set_tags(uid["2"], ["python", "frontend"])
@@ -1305,9 +1306,9 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) ->
store.add_company("Hidden", key="hidden")
agent = store.save_agent("A")
store.set_affinity(agent.id, "hidden", -1)
store.upsert_role(posting("1", company="Acme"))
store.upsert_role(posting("2", company="Acme"))
store.upsert_role(posting("3", company="Hidden"))
upsert(store, posting("1", company="Acme"))
upsert(store, posting("2", company="Acme"))
upsert(store, posting("3", company="Hidden"))
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
store.set_tags(uid["1"], ["python", "backend"])
store.set_tags(uid["2"], ["python"])
@@ -1320,7 +1321,7 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) ->
def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Store) -> None:
store.upsert_role(posting("a"))
upsert(store, posting("a"))
uid = store.roles()[0].uid
agent = store.save_agent("A")
store.delete_agent(agent.id)
@@ -1330,7 +1331,7 @@ def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Sto
def test_a_vote_moves_the_served_score_and_the_order(store: Store) -> None:
for sid in ("a", "b", "c"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
agent = store.save_agent("A")
other = store.save_agent("B")
@@ -1354,13 +1355,13 @@ def test_mark_closed_closes_what_the_board_no_longer_lists(store: Store) -> None
sub = store.add_subscription("ats", "greenhouse:acme")
other = store.add_subscription("ats", "greenhouse:other")
for source_id in ("1", "2", "3"):
store.upsert_role(posting(source_id), sub)
store.upsert_role(posting("9"), other)
upsert(store, posting(source_id), sub)
upsert(store, posting("9"), other)
assert store.mark_closed("ats", ["1", "3"], sub.id) == 1
assert {r.source_id for r in store.roles()} == {"1", "3", "9"}
assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"}
# Listed again: open again.
store.upsert_role(posting("2"), sub)
upsert(store, posting("2"), sub)
assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"}
@@ -1368,7 +1369,7 @@ def test_an_item_without_a_source_is_refused(store: Store) -> None:
item = posting()
item.source = ""
with pytest.raises(ValueError, match="no source"):
store.upsert_role(item)
upsert(store, item)
# --------------------------------------------------------------- companies
@@ -1472,7 +1473,7 @@ def test_the_default_agent_is_the_lowest_enabled_id(store: Store) -> None:
def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None:
for sid, place, country in (("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
("kbh", "København K", ""), ("none", "", "")):
store.upsert_role(posting(sid, place=place, country=country))
upsert(store, posting(sid, place=place, country=country))
agent = store.save_agent("A")
uid = {r.source_id: r.uid for r in store.roles()}
@@ -1490,8 +1491,8 @@ def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None
def test_open_role_counts_and_company_boards(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme")
store.upsert_role(posting("1", company="Acme", place="Berlin"), sub)
store.upsert_role(posting("2", company="Acme", place="Munich"), sub)
upsert(store, posting("1", company="Acme", place="Berlin"), sub)
upsert(store, posting("2", company="Acme", place="Munich"), sub)
store.mark_closed("ats", ["1"], sub.id)
assert store.open_role_counts() == {"acme": 1}
assert store.company_boards() == {"acme": "greenhouse"}
@@ -1524,7 +1525,7 @@ def test_a_clash_on_replace_leaves_the_old_board_and_its_roles(store: Store) ->
store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme")
_, beta, _ = store.set_company_board("beta", "lever:beta", host="lever", name="Beta")
assert beta is not None
store.upsert_role(posting("1", company="Beta"), beta)
upsert(store, posting("1", company="Beta"), beta)
with pytest.raises(ValueError):
store.set_company_board("beta", "greenhouse:acme", host="greenhouse",
fields={"ats": "greenhouse"})
@@ -1539,8 +1540,8 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles(
name="Acme")
assert old is not None
store.update_subscription(old.id, fetch=False, target_id="gh-1")
store.upsert_role(posting("1", company="Acme"), old)
store.upsert_role(posting("2", company="Acme"), old)
upsert(store, posting("1", company="Acme"), old)
upsert(store, posting("2", company="Acme"), old)
company, new, closed = store.set_company_board("acme", "ashby:acme", host="ashby",
fields={"ats": "ashby"})
@@ -1550,7 +1551,7 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles(
"ashby:acme", "ashby", "", 1, 1)
assert company.ats == "ashby"
# The same board again closes nothing.
store.upsert_role(posting("3", company="Acme"), new)
upsert(store, posting("3", company="Acme"), new)
assert store.set_company_board("acme", "ashby:acme", host="ashby")[2] == 0
assert [r.source_id for r in store.roles()] == ["3"]
@@ -1560,7 +1561,7 @@ def test_detaching_a_board_hides_it_closes_its_roles_and_keeps_the_row(
_, sub, _ = store.set_company_board("acme", "greenhouse:acme", host="greenhouse",
name="Acme")
assert sub is not None
store.upsert_role(posting("1", company="Acme"), sub)
upsert(store, posting("1", company="Acme"), sub)
company, detached, closed = store.set_company_board("acme", "")
assert closed == 1 and company.key == "acme"
assert detached is not None
@@ -1627,7 +1628,7 @@ def test_events_are_append_only(store: Store) -> None:
def test_an_application_may_point_at_a_role(store: Store) -> None:
store.upsert_role(posting())
upsert(store, posting())
saved = store.save_application(Application(role_uid=posting().uid, status="applied"))
assert saved.role_uid == posting().uid
assert [e.status for e in store.application_events(saved.id)] == ["applied"]
@@ -1636,7 +1637,7 @@ def test_an_application_may_point_at_a_role(store: Store) -> None:
# -------------------------------------------------- summary and posting text
def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
store.upsert_role(posting())
upsert(store, posting())
role = store.set_article_text(posting().uid, "The whole posting.")
assert role is not None
assert role.article_text == "The whole posting." and role.article_at == store.now()
@@ -1644,7 +1645,7 @@ def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
def test_a_summary_is_written_once(store: Store) -> None:
store.upsert_role(posting())
upsert(store, posting())
uid = posting().uid
assert store.set_summary(uid, "First.") is True
assert store.set_summary(uid, "Second.") is False
@@ -1655,7 +1656,7 @@ def test_a_summary_is_written_once(store: Store) -> None:
def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer(
store: Store) -> None:
store.upsert_role(posting())
upsert(store, posting())
uid = posting().uid
assert store.set_summary(uid, "First.", ["Python", "backend"]) is True
role = store.role(uid)
@@ -1675,7 +1676,7 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
agent that hid it. An agent saved before v8 has `limits` of '{}', which
reads as the defaults."""
for sid in "abcd":
store.upsert_role(posting(sid))
upsert(store, posting(sid))
a, b, c, d = (posting(s).uid for s in "abcd")
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))