Extract gate columns from posting text with a Haiku ModelTask
roledesk job extract reads open, unanswered roles 20 to a call and fills place, country, remote, seniority, salary, currency, function, permit and languages, only into columns the board left empty. Schema v2 adds roles.permit, roles.languages and roles.extract_tries. Off by default. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b44f78ca7e
commit
4dfb8831aa
12 files changed
+578
-14
No files matched your search
@@ -30,6 +30,21 @@ from roledesk.storage import schema
|
||||
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
#: structured field wins over what a model read out of the prose, and the next
|
||||
#: upsert from that board would put it back anyway.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
|
||||
"permit")
|
||||
|
||||
#: How many replies may leave a role unanswered before `extract` stops
|
||||
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
||||
#: never answers is otherwise paid for on every pull for as long as it exists.
|
||||
GIVE_UP_AFTER = 3
|
||||
|
||||
#: A role `extract` has yet to read: never answered, still open, not given up.
|
||||
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
|
||||
f"AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes")
|
||||
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
|
||||
"show", "target_id")
|
||||
@@ -270,6 +285,74 @@ class Store:
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
||||
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def extract_pending(self) -> int:
|
||||
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def extract_waiting(self, limit: int,
|
||||
uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `extract` has yet to read, newest first. Given `uids`, only
|
||||
those of them still waiting: a batch asks again when it runs, since a
|
||||
role may have been answered since the task shared the work out."""
|
||||
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
|
||||
if uids is not None:
|
||||
sql += " AND uid IN (SELECT value FROM json_each(?))"
|
||||
params.append(json.dumps(list(uids)))
|
||||
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
|
||||
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
|
||||
|
||||
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
|
||||
"""Write what `extract` read for one role and stamp `extracted_at`.
|
||||
|
||||
Only into empty columns, and only a non-empty value: a board's own
|
||||
field is never replaced and nothing is ever blanked. `place` brings
|
||||
`place_key` with it. Returns how many columns were filled, or None
|
||||
for an unknown uid. Values are trusted to be in vocabulary; the
|
||||
job's `apply` is where a reply is checked."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
sets: dict[str, Any] = {}
|
||||
for name in _EXTRACT_TEXT:
|
||||
value = str(fields.get(name) or "")
|
||||
if value and not row[name]:
|
||||
sets[name] = value
|
||||
for name in _RAW_INT:
|
||||
number = _int_or_none(fields.get(name))
|
||||
if number is not None and row[name] is None:
|
||||
sets[name] = number
|
||||
languages = [str(code) for code in fields.get("languages") or []]
|
||||
if languages and json.loads(row["languages"] or "[]") == []:
|
||||
sets["languages"] = json.dumps(languages)
|
||||
filled = len(sets)
|
||||
if "place" in sets:
|
||||
sets["place_key"] = place_key(sets["place"])
|
||||
sets["extracted_at"] = self.now()
|
||||
conn.execute(
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
||||
(*sets.values(), uid))
|
||||
return filled
|
||||
|
||||
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
|
||||
"""One reply that left these roles unanswered. Counted only against
|
||||
those still waiting, so a role the reply did answer is not charged.
|
||||
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
|
||||
for `ModelTask.missed` and not kept: there is no Given up list yet."""
|
||||
if not uids:
|
||||
return 0
|
||||
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
|
||||
wanted = json.dumps(list(uids))
|
||||
with self.transaction() as conn:
|
||||
crossed = conn.execute(
|
||||
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
|
||||
(wanted, GIVE_UP_AFTER)).fetchone()[0]
|
||||
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
|
||||
f"WHERE {where}", (wanted,))
|
||||
return int(crossed)
|
||||
|
||||
# ------------------------------------------------------------ companies
|
||||
|
||||
def add_company(self, name: str, *, key: str = "", domain: str = "",
|
||||
|
||||
Reference in new issue
Block a user