Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
+15
-14
@@ -16,6 +16,7 @@ from app.config import Config
|
||||
from app.jobs import extract
|
||||
from app.models import Limits
|
||||
from app.storage import Store
|
||||
from prepped import upsert
|
||||
|
||||
|
||||
def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item:
|
||||
@@ -77,7 +78,7 @@ def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
|
||||
def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = posting("1").uid
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer(
|
||||
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
|
||||
@@ -100,7 +101,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
||||
def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
ranked(store, item)
|
||||
stub(monkeypatch, lambda roles: [answer(
|
||||
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
|
||||
@@ -116,13 +117,13 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
||||
|
||||
def test_the_next_upsert_keeps_what_extract_filled(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
|
||||
for r in roles])
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
|
||||
role = store.role(posting("1").uid)
|
||||
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
|
||||
@@ -131,7 +132,7 @@ def test_the_next_upsert_keeps_what_extract_filled(
|
||||
def test_closed_and_answered_roles_are_not_offered(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for sid in ("open", "closed", "done"):
|
||||
store.upsert_role(posting(sid))
|
||||
upsert(store, posting(sid))
|
||||
ranked(store, posting("open"), posting("closed"), posting("done"))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
|
||||
@@ -147,7 +148,7 @@ def test_closed_and_answered_roles_are_not_offered(
|
||||
|
||||
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
|
||||
|
||||
@@ -170,7 +171,7 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
the boards produced. A role nobody ranks is never sent; a batch dealt a
|
||||
uid finishes it."""
|
||||
for sid in ("best", "second", "low", "unscored", "gated"):
|
||||
store.upsert_role(posting(sid))
|
||||
upsert(store, posting(sid))
|
||||
one = store.save_agent("One", "python backend",
|
||||
limits=Limits(extract_top=1, extract_min=0.5))
|
||||
two = store.save_agent("Two", "python backend", enabled=False)
|
||||
@@ -209,7 +210,7 @@ def test_a_value_outside_the_vocabulary_is_dropped() -> None:
|
||||
|
||||
def test_disabled_finds_nothing_and_asks_nobody(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
model = stub(monkeypatch, lambda roles: [])
|
||||
|
||||
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
|
||||
@@ -220,7 +221,7 @@ def test_disabled_finds_nothing_and_asks_nobody(
|
||||
def test_a_role_read_before_without_a_level_is_read_again_for_it(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1", seniority="senior")
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
ranked(store, item)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||
@@ -243,8 +244,8 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
||||
withtext = posting("text", title="Senior Engineer", seniority="mid")
|
||||
noText = posting("teaser", title="Analyst")
|
||||
noText.body = ""
|
||||
store.upsert_role(withtext)
|
||||
store.upsert_role(noText)
|
||||
upsert(store, withtext)
|
||||
upsert(store, noText)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
|
||||
"WHERE source_id = 'teaser'")
|
||||
@@ -265,7 +266,7 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
||||
def test_an_answer_with_no_level_is_charged_until_given_up(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1")
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
ranked(store, item)
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
|
||||
|
||||
@@ -279,7 +280,7 @@ def test_an_answer_with_no_level_is_charged_until_given_up(
|
||||
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
|
||||
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
|
||||
for item in items:
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
one = store.save_agent("One", "python backend")
|
||||
two = store.save_agent("Two", "python backend")
|
||||
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
|
||||
@@ -296,7 +297,7 @@ def test_agreement_per_host(store: Store) -> None:
|
||||
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
|
||||
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
|
||||
("5", "lever", "senior", "")]:
|
||||
store.upsert_role(posting(sid, board=board, seniority=stated))
|
||||
upsert(store, posting(sid, board=board, seniority=stated))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
|
||||
(model, sid))
|
||||
|
||||
Reference in new issue
Block a user