A pull that brings a role a better text, as fetch text does, puts it back in extract's queue beside dropping its vector, so a role extract read from its title alone is read again from its text.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
7ac63dea7c
commit
0b92f19f4d
3 files changed
+77
-10
No files matched your search
+19
-10
@@ -65,7 +65,8 @@ _EXTRACT_TEXT = ("place", "country", "currency", "permit")
|
|||||||
GIVE_UP_AFTER = 3
|
GIVE_UP_AFTER = 3
|
||||||
|
|
||||||
#: A role `extract` has yet to read: still open, no copy of another
|
#: A role `extract` has yet to read: still open, no copy of another
|
||||||
#: (`rebuild_dupes`), not given up, and either never answered, without a
|
#: (`rebuild_dupes`), not given up, and either never answered (or answered
|
||||||
|
#: before a better text came, `upsert_role`), without a
|
||||||
#: model level, never asked where a remote hire may live
|
#: model level, never asked where a remote hire may live
|
||||||
#: (`remote_where_model` NULL), or never asked for its languages and deadline
|
#: (`remote_where_model` NULL), or never asked for its languages and deadline
|
||||||
#: (`languages_required` NULL). The last three make `extract` the
|
#: (`languages_required` NULL). The last three make `extract` the
|
||||||
@@ -835,8 +836,9 @@ class Store:
|
|||||||
the way a board's own structured field is (docs/decisions.md, "a
|
the way a board's own structured field is (docs/decisions.md, "a
|
||||||
guess is not a board field"). So once a role has been through
|
guess is not a board field"). So once a role has been through
|
||||||
`extract` (`extracted_at` is set), this upsert leaves `function`
|
`extract` (`extracted_at` is set), this upsert leaves `function`
|
||||||
alone for good, whatever the plugin's raw guess now says — see
|
alone, whatever the plugin's raw guess now says, until a better text
|
||||||
`set_extracted` for the other half of the rule.
|
puts the role back in extract's queue (below) — see `set_extracted`
|
||||||
|
for the other half of the rule.
|
||||||
|
|
||||||
The role's places (`role_places`) are the texts of `raw["places"]`,
|
The role's places (`role_places`) are the texts of `raw["places"]`,
|
||||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
||||||
@@ -848,9 +850,13 @@ class Store:
|
|||||||
under the same keep-filled rule. `teaser` is kept-filled too.
|
under the same keep-filled rule. `teaser` is kept-filled too.
|
||||||
|
|
||||||
A role is embedded from its description, else its teaser, else its
|
A role is embedded from its description, else its teaser, else its
|
||||||
title (`app.jobs.embed.document`). When a pull brings a better one of
|
title (`app.jobs.embed.document`), and `extract` reads the same. When
|
||||||
these than the role had, its vector is dropped so the next `embed`
|
a pull brings a better one of these than the role had, its vector is
|
||||||
reads it.
|
dropped so the next `embed` reads it, and `extracted_at` and
|
||||||
|
`extract_tries` are reset so `extract` reads it again (if it is still
|
||||||
|
in a cut). That read overwrites the model's own columns and fills
|
||||||
|
what is still empty (`set_extracted`); until it comes, `function` is
|
||||||
|
the plugin's guess's again, as for any role not yet read.
|
||||||
"""
|
"""
|
||||||
if not item.source:
|
if not item.source:
|
||||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||||
@@ -937,14 +943,17 @@ class Store:
|
|||||||
if before is not None:
|
if before is not None:
|
||||||
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
||||||
(row["uid"],))
|
(row["uid"],))
|
||||||
# `embed` encodes a role once, from the best text it has. When a
|
# `embed` encodes a role once, from the best text it has, and
|
||||||
# later pull brings a better one, as a host that states the text
|
# `extract` reads it once. When a later pull brings a better one,
|
||||||
# on the role's own page does, the vector goes, so the next
|
# as fetch text does for a host that states the text on the
|
||||||
# `embed` reads it.
|
# role's own page, the vector goes and the role waits for
|
||||||
|
# extract again, so both read the new text.
|
||||||
if before is not None and _text_rank(row["description"] or before["description"],
|
if before is not None and _text_rank(row["description"] or before["description"],
|
||||||
row["teaser"] or before["teaser"]) \
|
row["teaser"] or before["teaser"]) \
|
||||||
> _text_rank(before["description"], before["teaser"]):
|
> _text_rank(before["description"], before["teaser"]):
|
||||||
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
|
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
|
||||||
|
conn.execute("UPDATE roles SET extracted_at = NULL, extract_tries = 0 "
|
||||||
|
"WHERE uid=?", (row["uid"],))
|
||||||
return before is None
|
return before is None
|
||||||
|
|
||||||
def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None:
|
def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None:
|
||||||
|
|||||||
@@ -130,6 +130,46 @@ def test_the_next_upsert_keeps_what_extract_filled(
|
|||||||
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
|
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_read_from_its_title_is_read_again_once_its_text_arrives(
|
||||||
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""Fetch text stores what a page filled through `upsert_role`: a better
|
||||||
|
text than the role was read from puts it back in extract's queue, its
|
||||||
|
tries forgiven, and a pull that brings no better text does not."""
|
||||||
|
bare = posting("1")
|
||||||
|
bare.body = ""
|
||||||
|
upsert(store, bare)
|
||||||
|
ranked(store, bare)
|
||||||
|
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid", function="sales")
|
||||||
|
for r in roles])
|
||||||
|
modeltask.run(extract.TASK, config, store)
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET extract_tries = 2 WHERE source_id = '1'")
|
||||||
|
|
||||||
|
upsert(store, bare)
|
||||||
|
assert store.extract_pending() == 0, "the same text again is no reason to read it"
|
||||||
|
|
||||||
|
upsert(store, posting("1", function="engineering"))
|
||||||
|
|
||||||
|
role = store.role(bare.uid)
|
||||||
|
assert role is not None and role.extracted_at is None
|
||||||
|
assert role.function == "sales", "the pull that brings the text keeps extract's function"
|
||||||
|
assert [r.uid for r in store.extract_waiting(10)] == [bare.uid]
|
||||||
|
with store.transaction() as conn:
|
||||||
|
assert conn.execute("SELECT extract_tries FROM roles WHERE uid = ?",
|
||||||
|
(bare.uid,)).fetchone()[0] == 0
|
||||||
|
model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="senior")
|
||||||
|
for r in roles])
|
||||||
|
modeltask.run(extract.TASK, config, store)
|
||||||
|
[[sent]] = model.asked
|
||||||
|
assert sent["text"] == "We pay 80.000-95.000 EUR. Hybrid in Berlin."
|
||||||
|
role = store.role(bare.uid)
|
||||||
|
assert role is not None and role.extracted_at is not None
|
||||||
|
assert (role.seniority_model, role.seniority) == ("senior", "senior")
|
||||||
|
|
||||||
|
upsert(store, posting("1"))
|
||||||
|
assert store.extract_pending() == 0, "a text no better than before is read once"
|
||||||
|
|
||||||
|
|
||||||
def test_closed_and_answered_roles_are_not_offered(
|
def test_closed_and_answered_roles_are_not_offered(
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
for sid in ("open", "closed", "done"):
|
for sid in ("open", "closed", "done"):
|
||||||
|
|||||||
@@ -237,6 +237,24 @@ def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors(
|
|||||||
assert again.place == "Copenhagen"
|
assert again.place == "Copenhagen"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_extract_read_without_text_waits_for_it_again_once_its_page_is_read(
|
||||||
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
|
"""Extract read the title alone; the page's text puts the role back in
|
||||||
|
its queue, and a page that gave nothing leaves it read."""
|
||||||
|
board(store, "fake:acme")
|
||||||
|
filled, empty = (role(store, "fake:acme", n) for n in ("1", "2"))
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||||
|
"extract_tries = 1")
|
||||||
|
web["answers"].update({f"{BASE}/acme/1": b"Python in full"})
|
||||||
|
|
||||||
|
fetch_text.run_batch(config(tmp_path), store, target="fake:acme", uids=[filled, empty])
|
||||||
|
|
||||||
|
first, second = store.role(filled), store.role(empty)
|
||||||
|
assert first is not None and first.extracted_at is None
|
||||||
|
assert second is not None and second.extracted_at == "2026-09-22T00:00:00+00:00"
|
||||||
|
|
||||||
|
|
||||||
def test_a_board_switched_off_since_the_plan_reads_nothing(
|
def test_a_board_switched_off_since_the_plan_reads_nothing(
|
||||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||||
sub = board(store, "fake:acme")
|
sub = board(store, "fake:acme")
|
||||||
|
|||||||
Reference in new issue
Block a user