Embed a role again when its text arrives, and keep the place its own page gave

A role listed before its page was read kept a title-only vector, and the listing's place replaced the page's on every later pull.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-29 22:23:43 +02:00
1 parent fd89381caa
commit 31abad4c8e
5 files changed
+69 -6

No files matched your search

+9 -3
View File
@@ -415,12 +415,18 @@ class Store:
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
with self.transaction() as conn:
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
(row["uid"],)).fetchone() is not None
before = conn.execute("SELECT description FROM roles WHERE uid=?",
(row["uid"],)).fetchone()
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
return not existed
# A role listed without its text is embedded from its title, and
# `embed` encodes a role once. When the text arrives on a later
# pull, as it does for a host that states it on the role's own
# page, the vector goes, so the next `embed` reads the text.
if before is not None and not before["description"] and row["description"]:
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
return before is None
def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None:
"""One role with its tags, and with `vote`, `voted_at`, `score`,
+16 -3
View File
@@ -618,7 +618,9 @@ DETAIL_CAP = 200
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
#: earliest date a role ever had. `title` and `url` are not here, since the
#: earliest date a role ever had. A filled column is still overwritten by
#: another filled value, so `_read_details` withholds the listing's place and
#: remote for a role already read. `title` and `url` are not here, since the
#: listing always states them and would overwrite them; nor are `department`
#: (no column of its own, only `raw` and the classifier's guess) and
#: `salary_text`, both rewritten from the listing on every pull.
@@ -641,7 +643,10 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
is left out with a debug line. Every row not read in detail in this call
goes out with an empty description: a role read on an earlier pull keeps
the one it has, since `upsert_role` never overwrites a filled column with
an empty value, and a teaser from the listing must not replace it.
an empty value, and a teaser from the listing must not replace it. A role
read on an earlier pull goes out without the listing's place and remote
as well, for the same reason: the listing states them every pull
("2 Locations"), and would replace what the role's own page said.
A detail that answers 404 or 410 counts as read: the role is gone or has
no page of its own, and asking again would not change that. One that
@@ -691,7 +696,15 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
done.add(key)
failing.discard(key)
out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows]
out = []
for row in rows:
key = compound(row)
if key in filled:
out.append(row | {"description": ""} | filled[key])
elif key in detailed:
out.append(row | {"description": "", "location": "", "remote": None})
else:
out.append(row | {"description": ""})
if complete:
listed = {compound(row) for row in rows}
done &= listed
+2
View File
@@ -113,6 +113,8 @@ listing's empty value; any other key is left out with a debug log line.
on a pull is sent with an empty description, so the one read earlier stays.
A 404 or a 410 counts as read; any other failure is logged and retried on a
later pull; a 429 ends the detail reads of that board for the pull.
A role read in detail on an earlier pull is sent without the
listing's place and remote, so what its own page said about them stays.
Completeness is decided by the listing alone.
**robots.txt.** With `robots_page(slug)`, the shared code reads
+24
View File
@@ -328,6 +328,30 @@ def test_detail_is_read_for_unread_roles_only(
assert state["ats:complete:fake:acme"] == "1"
def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The listing states a place on every pull, and a filled value replaces
a filled column, so for a role already read it is withheld."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full",
f"{base}/3": RuntimeError("down")})
module = _fake_host(detail=lambda get, slug, row: {
"description": get(f"{base}/{row['source_id']}").decode()})
listing = module.parse
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
for row in listing(body, slug)]
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
items, _state = _run(monkeypatch, module, state)
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
assert by_id["fake:acme:1"] == ("", "", None)
# Read now, and its page stated neither: the listing's stand.
assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid")
# Never read: the listing's stand until its page is.
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
+18
View File
@@ -549,6 +549,24 @@ def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
("acme", "senior backend engineer", "berlin")
def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None:
listed = posting()
listed.body = ""
store.upsert_role(listed)
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
assert store.roles_to_embed("m", 10) == []
store.upsert_role(listed) # listed again, still without text
assert store.roles_to_embed("m", 10) == []
store.upsert_role(posting()) # the role's own page was read
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
store.upsert_role(listed) # later pulls send no text
store.upsert_role(posting()) # or the same text again
assert store.roles_to_embed("m", 10) == []
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
salary_max=95000, currency="EUR", department="Eng"))