Embed a role again when its text arrives, and keep the place its own page gave
A role listed before its page was read kept a title-only vector, and the listing's place replaced the page's on every later pull. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
fd89381caa
commit
31abad4c8e
5 files changed
+69
-6
No files matched your search
@@ -415,12 +415,18 @@ class Store:
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
with self.transaction() as conn:
|
||||
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
|
||||
(row["uid"],)).fetchone() is not None
|
||||
before = conn.execute("SELECT description FROM roles WHERE uid=?",
|
||||
(row["uid"],)).fetchone()
|
||||
conn.execute(
|
||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||
return not existed
|
||||
# A role listed without its text is embedded from its title, and
|
||||
# `embed` encodes a role once. When the text arrives on a later
|
||||
# pull, as it does for a host that states it on the role's own
|
||||
# page, the vector goes, so the next `embed` reads the text.
|
||||
if before is not None and not before["description"] and row["description"]:
|
||||
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
|
||||
return before is None
|
||||
|
||||
def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None:
|
||||
"""One role with its tags, and with `vote`, `voted_at`, `score`,
|
||||
|
||||
+16
-3
@@ -618,7 +618,9 @@ DETAIL_CAP = 200
|
||||
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
|
||||
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
|
||||
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
|
||||
#: earliest date a role ever had. `title` and `url` are not here, since the
|
||||
#: earliest date a role ever had. A filled column is still overwritten by
|
||||
#: another filled value, so `_read_details` withholds the listing's place and
|
||||
#: remote for a role already read. `title` and `url` are not here, since the
|
||||
#: listing always states them and would overwrite them; nor are `department`
|
||||
#: (no column of its own, only `raw` and the classifier's guess) and
|
||||
#: `salary_text`, both rewritten from the listing on every pull.
|
||||
@@ -641,7 +643,10 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
|
||||
is left out with a debug line. Every row not read in detail in this call
|
||||
goes out with an empty description: a role read on an earlier pull keeps
|
||||
the one it has, since `upsert_role` never overwrites a filled column with
|
||||
an empty value, and a teaser from the listing must not replace it.
|
||||
an empty value, and a teaser from the listing must not replace it. A role
|
||||
read on an earlier pull goes out without the listing's place and remote
|
||||
as well, for the same reason: the listing states them every pull
|
||||
("2 Locations"), and would replace what the role's own page said.
|
||||
|
||||
A detail that answers 404 or 410 counts as read: the role is gone or has
|
||||
no page of its own, and asking again would not change that. One that
|
||||
@@ -691,7 +696,15 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
|
||||
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows]
|
||||
out = []
|
||||
for row in rows:
|
||||
key = compound(row)
|
||||
if key in filled:
|
||||
out.append(row | {"description": ""} | filled[key])
|
||||
elif key in detailed:
|
||||
out.append(row | {"description": "", "location": "", "remote": None})
|
||||
else:
|
||||
out.append(row | {"description": ""})
|
||||
if complete:
|
||||
listed = {compound(row) for row in rows}
|
||||
done &= listed
|
||||
|
||||
@@ -113,6 +113,8 @@ listing's empty value; any other key is left out with a debug log line.
|
||||
on a pull is sent with an empty description, so the one read earlier stays.
|
||||
A 404 or a 410 counts as read; any other failure is logged and retried on a
|
||||
later pull; a 429 ends the detail reads of that board for the pull.
|
||||
A role read in detail on an earlier pull is sent without the
|
||||
listing's place and remote, so what its own page said about them stays.
|
||||
Completeness is decided by the listing alone.
|
||||
|
||||
**robots.txt.** With `robots_page(slug)`, the shared code reads
|
||||
|
||||
@@ -328,6 +328,30 @@ def test_detail_is_read_for_unread_roles_only(
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The listing states a place on every pull, and a filled value replaces
|
||||
a filled column, so for a role already read it is withheld."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full",
|
||||
f"{base}/3": RuntimeError("down")})
|
||||
module = _fake_host(detail=lambda get, slug, row: {
|
||||
"description": get(f"{base}/{row['source_id']}").decode()})
|
||||
listing = module.parse
|
||||
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
|
||||
for row in listing(body, slug)]
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"]),
|
||||
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
|
||||
items, _state = _run(monkeypatch, module, state)
|
||||
|
||||
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
|
||||
assert by_id["fake:acme:1"] == ("", "", None)
|
||||
# Read now, and its page stated neither: the listing's stand.
|
||||
assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid")
|
||||
# Never read: the listing's stand until its page is.
|
||||
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
|
||||
|
||||
|
||||
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
|
||||
@@ -549,6 +549,24 @@ def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
|
||||
("acme", "senior backend engineer", "berlin")
|
||||
|
||||
|
||||
def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None:
|
||||
listed = posting()
|
||||
listed.body = ""
|
||||
store.upsert_role(listed)
|
||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
|
||||
store.upsert_role(listed) # listed again, still without text
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
store.upsert_role(posting()) # the role's own page was read
|
||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||
|
||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||
store.upsert_role(listed) # later pulls send no text
|
||||
store.upsert_role(posting()) # or the same text again
|
||||
assert store.roles_to_embed("m", 10) == []
|
||||
|
||||
|
||||
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
||||
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
|
||||
salary_max=95000, currency="EUR", department="Eng"))
|
||||
|
||||
Reference in new issue
Block a user