Extract gate columns from posting text with a Haiku ModelTask

roledesk job extract reads open, unanswered roles 20 to a call and fills
place, country, remote, seniority, salary, currency, function, permit and
languages, only into columns the board left empty. Schema v2 adds
roles.permit, roles.languages and roles.extract_tries. Off by default.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:05:30 +02:00
1 parent b44f78ca7e
commit 4dfb8831aa
12 files changed
+578 -14

No files matched your search

+83
View File
@@ -30,6 +30,21 @@ from roledesk.storage import schema
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
#: structured field wins over what a model read out of the prose, and the next
#: upsert from that board would put it back anyway.
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
"permit")
#: How many replies may leave a role unanswered before `extract` stops
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
#: never answers is otherwise paid for on every pull for as long as it exists.
GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: never answered, still open, not given up.
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
f"AND extract_tries < {GIVE_UP_AFTER}")
_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes")
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
"show", "target_id")
@@ -270,6 +285,74 @@ class Store:
AND source_id NOT IN (SELECT value FROM json_each(?))""",
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
# -------------------------------------------------------------- extract
def extract_pending(self) -> int:
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
return int(row["n"]) if row else 0
def extract_waiting(self, limit: int,
uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `extract` has yet to read, newest first. Given `uids`, only
those of them still waiting: a batch asks again when it runs, since a
role may have been answered since the task shared the work out."""
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
if uids is not None:
sql += " AND uid IN (SELECT value FROM json_each(?))"
params.append(json.dumps(list(uids)))
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
"""Write what `extract` read for one role and stamp `extracted_at`.
Only into empty columns, and only a non-empty value: a board's own
field is never replaced and nothing is ever blanked. `place` brings
`place_key` with it. Returns how many columns were filled, or None
for an unknown uid. Values are trusted to be in vocabulary; the
job's `apply` is where a reply is checked."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
return None
sets: dict[str, Any] = {}
for name in _EXTRACT_TEXT:
value = str(fields.get(name) or "")
if value and not row[name]:
sets[name] = value
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
sets[name] = number
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
(*sets.values(), uid))
return filled
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
"""One reply that left these roles unanswered. Counted only against
those still waiting, so a role the reply did answer is not charged.
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
for `ModelTask.missed` and not kept: there is no Given up list yet."""
if not uids:
return 0
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
wanted = json.dumps(list(uids))
with self.transaction() as conn:
crossed = conn.execute(
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
(wanted, GIVE_UP_AFTER)).fetchone()[0]
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
f"WHERE {where}", (wanted,))
return int(crossed)
# ------------------------------------------------------------ companies
def add_company(self, name: str, *, key: str = "", domain: str = "",