From 0b92f19f4d45925e0eb2b6f0d19e83e7d4cfba80 Mon Sep 17 00:00:00 2001 From: Wolf Date: Mon, 5 Oct 2026 09:30:17 +0200 Subject: [PATCH] A pull that brings a role a better text, as fetch text does, puts it back in extract's queue beside dropping its vector, so a role extract read from its title alone is read again from its text. Co-Authored-By: Claude Fable 5.1 --- app/storage/store.py | 29 +++++++++++++++++++---------- tests/test_extract.py | 40 ++++++++++++++++++++++++++++++++++++++++ tests/test_fetch_text.py | 18 ++++++++++++++++++ 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/app/storage/store.py b/app/storage/store.py index 5d7f75b..4b1b01d 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -65,7 +65,8 @@ _EXTRACT_TEXT = ("place", "country", "currency", "permit") GIVE_UP_AFTER = 3 #: A role `extract` has yet to read: still open, no copy of another -#: (`rebuild_dupes`), not given up, and either never answered, without a +#: (`rebuild_dupes`), not given up, and either never answered (or answered +#: before a better text came, `upsert_role`), without a #: model level, never asked where a remote hire may live #: (`remote_where_model` NULL), or never asked for its languages and deadline #: (`languages_required` NULL). The last three make `extract` the @@ -835,8 +836,9 @@ class Store: the way a board's own structured field is (docs/decisions.md, "a guess is not a board field"). So once a role has been through `extract` (`extracted_at` is set), this upsert leaves `function` - alone for good, whatever the plugin's raw guess now says — see - `set_extracted` for the other half of the rule. + alone, whatever the plugin's raw guess now says, until a better text + puts the role back in extract's queue (below) — see `set_extracted` + for the other half of the rule. The role's places (`role_places`) are the texts of `raw["places"]`, else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved @@ -848,9 +850,13 @@ class Store: under the same keep-filled rule. `teaser` is kept-filled too. A role is embedded from its description, else its teaser, else its - title (`app.jobs.embed.document`). When a pull brings a better one of - these than the role had, its vector is dropped so the next `embed` - reads it. + title (`app.jobs.embed.document`), and `extract` reads the same. When + a pull brings a better one of these than the role had, its vector is + dropped so the next `embed` reads it, and `extracted_at` and + `extract_tries` are reset so `extract` reads it again (if it is still + in a cut). That read overwrites the model's own columns and fills + what is still empty (`set_extracted`); until it comes, `function` is + the plugin's guess's again, as for any role not yet read. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") @@ -937,14 +943,17 @@ class Store: if before is not None: conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?", (row["uid"],)) - # `embed` encodes a role once, from the best text it has. When a - # later pull brings a better one, as a host that states the text - # on the role's own page does, the vector goes, so the next - # `embed` reads it. + # `embed` encodes a role once, from the best text it has, and + # `extract` reads it once. When a later pull brings a better one, + # as fetch text does for a host that states the text on the + # role's own page, the vector goes and the role waits for + # extract again, so both read the new text. if before is not None and _text_rank(row["description"] or before["description"], row["teaser"] or before["teaser"]) \ > _text_rank(before["description"], before["teaser"]): conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],)) + conn.execute("UPDATE roles SET extracted_at = NULL, extract_tries = 0 " + "WHERE uid=?", (row["uid"],)) return before is None def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None: diff --git a/tests/test_extract.py b/tests/test_extract.py index 951213a..b6062d0 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -130,6 +130,46 @@ def test_the_next_upsert_keeps_what_extract_filled( assert role is not None and (role.country, role.seniority) == ("DE", "mid") +def test_a_role_read_from_its_title_is_read_again_once_its_text_arrives( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + """Fetch text stores what a page filled through `upsert_role`: a better + text than the role was read from puts it back in extract's queue, its + tries forgiven, and a pull that brings no better text does not.""" + bare = posting("1") + bare.body = "" + upsert(store, bare) + ranked(store, bare) + stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid", function="sales") + for r in roles]) + modeltask.run(extract.TASK, config, store) + with store.transaction() as conn: + conn.execute("UPDATE roles SET extract_tries = 2 WHERE source_id = '1'") + + upsert(store, bare) + assert store.extract_pending() == 0, "the same text again is no reason to read it" + + upsert(store, posting("1", function="engineering")) + + role = store.role(bare.uid) + assert role is not None and role.extracted_at is None + assert role.function == "sales", "the pull that brings the text keeps extract's function" + assert [r.uid for r in store.extract_waiting(10)] == [bare.uid] + with store.transaction() as conn: + assert conn.execute("SELECT extract_tries FROM roles WHERE uid = ?", + (bare.uid,)).fetchone()[0] == 0 + model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="senior") + for r in roles]) + modeltask.run(extract.TASK, config, store) + [[sent]] = model.asked + assert sent["text"] == "We pay 80.000-95.000 EUR. Hybrid in Berlin." + role = store.role(bare.uid) + assert role is not None and role.extracted_at is not None + assert (role.seniority_model, role.seniority) == ("senior", "senior") + + upsert(store, posting("1")) + assert store.extract_pending() == 0, "a text no better than before is read once" + + def test_closed_and_answered_roles_are_not_offered( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: for sid in ("open", "closed", "done"): diff --git a/tests/test_fetch_text.py b/tests/test_fetch_text.py index aeba2e6..fe9b77d 100644 --- a/tests/test_fetch_text.py +++ b/tests/test_fetch_text.py @@ -237,6 +237,24 @@ def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors( assert again.place == "Copenhagen" +def test_a_role_extract_read_without_text_waits_for_it_again_once_its_page_is_read( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + """Extract read the title alone; the page's text puts the role back in + its queue, and a page that gave nothing leaves it read.""" + board(store, "fake:acme") + filled, empty = (role(store, "fake:acme", n) for n in ("1", "2")) + with store.transaction() as conn: + conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', " + "extract_tries = 1") + web["answers"].update({f"{BASE}/acme/1": b"Python in full"}) + + fetch_text.run_batch(config(tmp_path), store, target="fake:acme", uids=[filled, empty]) + + first, second = store.role(filled), store.role(empty) + assert first is not None and first.extracted_at is None + assert second is not None and second.extracted_at == "2026-09-22T00:00:00+00:00" + + def test_a_board_switched_off_since_the_plan_reads_nothing( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: sub = board(store, "fake:acme")