Give an agent a CV, languages, avoid and must texts, and read a posting's languages with a level and its deadline (migration 28)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-10-03 21:54:16 +02:00
1 parent ed6d641373
commit 4fc474c437
15 files changed
+716 -79

No files matched your search

+100 -7
View File
@@ -19,7 +19,8 @@ from collections.abc import Callable
from app import places
from app.models import (
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
SENIORITIES, company_fold, company_key, country_of, host_word, iso_day, posted_at,
posted_host,
)
from app.seniority import from_title
from app.storage import gazetteer
@@ -120,7 +121,11 @@ CREATE TABLE IF NOT EXISTS roles (
employment TEXT NOT NULL DEFAULT '',
prep_due INTEGER NOT NULL DEFAULT 0,
remote_model TEXT NOT NULL DEFAULT '',
remote_where_model TEXT
remote_where_model TEXT,
languages_required TEXT,
deadline_host TEXT NOT NULL DEFAULT '',
deadline_model TEXT NOT NULL DEFAULT '',
deadline TEXT
);
-- The *_host columns hold the host's own words as a reader handed them on
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
@@ -134,9 +139,14 @@ CREATE TABLE IF NOT EXISTS roles (
-- word mapped to app.seniority's vocabulary, and the title's. remote_model is
-- extract's remote, which wins over the host's word and the place line, and
-- remote_where_model the names it read for where a remote hire may live (a
-- JSON list, NULL never asked, [] not stated). No comment sits among the last
-- columns: SQLite cannot drop a column a comment precedes, which the
-- migration tests do.
-- JSON list, NULL never asked, [] not stated). languages_required is every
-- language extract read in the posting, a JSON list of {language, level,
-- kind}, NULL never asked; languages keeps the required codes for the
-- languages gate. deadline_host is the host's own last day to apply, as a
-- pull hands it on, deadline_model the one extract read, both YYYY-MM-DD or
-- '', and deadline the first of them, NULL for none, written by prep. No
-- comment sits among the last columns: SQLite cannot drop a column a comment
-- precedes, which the migration tests do.
-- The gazetteer: a place is a continent, a country (key the ISO code), a
-- region or a city, in the place parent_id names. key is unique per kind:
@@ -317,7 +327,10 @@ END;
-- limits is how deep into this agent's ranking extract and summarise read,
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
-- means no narrowing.
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
-- level}), avoid_text (what drains) and must_text (must-haves and
-- deal-breakers) are what judge reads a role against; an agent with no CV
-- is not judged.
CREATE TABLE IF NOT EXISTS agents (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
@@ -326,7 +339,11 @@ CREATE TABLE IF NOT EXISTS agents (
enabled INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL,
limits TEXT NOT NULL DEFAULT '{}',
fetch_terms TEXT NOT NULL DEFAULT '[]'
fetch_terms TEXT NOT NULL DEFAULT '[]',
cv_text TEXT NOT NULL DEFAULT '',
cv_languages TEXT NOT NULL DEFAULT '[]',
avoid_text TEXT NOT NULL DEFAULT '',
must_text TEXT NOT NULL DEFAULT ''
);
-- What an agent looks for, as several named texts in order; a role's fit is
@@ -378,6 +395,29 @@ CREATE TABLE IF NOT EXISTS scores (
PRIMARY KEY (uid, agent_id)
);
-- One agent's judge reading of a role against its CV. Its own table, since
-- replace_scores rewrites scores every run and a judgement outlives that
-- and the role closing. skills, experience and career are 0 to 100,
-- cv_score their weighted total, strengths and gaps JSON lists. input_key
-- is a hash of the CV, prompts, avoid text and the role's text it read: a
-- different one is read again. tries counts replies that left the role
-- without one since; at 3 it is given up until the agent is saved again. A
-- row with judged_at NULL holds only tries.
CREATE TABLE IF NOT EXISTS role_judgements (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
skills INTEGER,
experience INTEGER,
career INTEGER,
cv_score REAL,
strengths TEXT NOT NULL DEFAULT '[]',
gaps TEXT NOT NULL DEFAULT '[]',
input_key TEXT NOT NULL DEFAULT '',
tries INTEGER NOT NULL DEFAULT 0,
judged_at TEXT,
PRIMARY KEY (uid, agent_id)
);
-- One vector per role, little-endian float32. Only comparable to another from
-- the same model, so every read is scoped by it.
CREATE TABLE IF NOT EXISTS vectors (
@@ -442,6 +482,10 @@ CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
CREATE INDEX IF NOT EXISTS idx_role_remote_place ON role_remote(place_id, uid);
-- The roles the next prep maps. Partial: after a prep there are none.
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
-- The feed's Closing soon filter. Partial: most roles state no deadline.
CREATE INDEX IF NOT EXISTS idx_roles_deadline ON roles(deadline) WHERE deadline IS NOT NULL;
-- An agent's judgements, for its feed under Best CV match.
CREATE INDEX IF NOT EXISTS idx_role_judgements_agent ON role_judgements(agent_id, cv_score DESC);
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -649,6 +693,11 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
# Anywhere. The areas are written by the next prep (app.prep.stage.VERSION 3).
(27, "roles.remote_model and roles.remote_where_model, role_remote and the Anywhere "
"place: where a remote hire may live", lambda conn: _add_remote_area(conn)),
# A Python step: it adds the columns only where they are missing. The
# deadline is composed by the next prep (app.prep.stage.VERSION 4).
(28, "agents.cv_text, cv_languages, avoid_text and must_text, roles.languages_required and the "
"deadline columns, and role_judgements: the CV match, the language flag and the "
"deadline", lambda conn: _add_cv_match(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -932,6 +981,50 @@ def _add_remote_area(conn: sqlite3.Connection) -> None:
gazetteer.seed(conn)
def _add_cv_match(conn: sqlite3.Connection) -> None:
"""Migration 28's step: an agent's CV, languages, avoid text and
must-haves, all empty; `roles.languages_required` NULL for every role, so `extract`
reads the roles it has read once more for the languages and the
deadline; `roles.deadline_host` from the date a reader already keeps in
`raw` (`applicationDeadline`, Jobnet and Jobbank), `deadline_model`
empty and `deadline` NULL until the next prep; and `role_judgements`,
empty. Runs twice without harm."""
agents = {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
for column, ddl in (("cv_text", "TEXT NOT NULL DEFAULT ''"),
("cv_languages", "TEXT NOT NULL DEFAULT '[]'"),
("avoid_text", "TEXT NOT NULL DEFAULT ''"),
("must_text", "TEXT NOT NULL DEFAULT ''")):
if column not in agents:
conn.execute(f"ALTER TABLE agents ADD COLUMN {column} {ddl}")
roles = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
for column, ddl in (("languages_required", "TEXT"),
("deadline_host", "TEXT NOT NULL DEFAULT ''"),
("deadline_model", "TEXT NOT NULL DEFAULT ''"),
("deadline", "TEXT")):
if column not in roles:
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} {ddl}")
dated = conn.execute(
"SELECT uid, json_extract(raw, '$.applicationDeadline') FROM roles "
"WHERE deadline_host = '' AND json_extract(raw, '$.applicationDeadline') IS NOT NULL"
).fetchall()
found = [(day, uid) for uid, value in dated if (day := iso_day(value))]
conn.executemany("UPDATE roles SET deadline_host = ? WHERE uid = ?", found)
log.info("migration 28: a host deadline on %d roles", len(found))
conn.execute("""CREATE TABLE IF NOT EXISTS role_judgements (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
skills INTEGER,
experience INTEGER,
career INTEGER,
cv_score REAL,
strengths TEXT NOT NULL DEFAULT '[]',
gaps TEXT NOT NULL DEFAULT '[]',
input_key TEXT NOT NULL DEFAULT '',
tries INTEGER NOT NULL DEFAULT 0,
judged_at TEXT,
PRIMARY KEY (uid, agent_id))""")
def _add_all_roles(conn: sqlite3.Connection) -> None:
"""Migration 21's step. The column; the tag for every company that is a
favourite of any agent, since favourite no longer widens what a pull
+66 -26
View File
@@ -24,10 +24,11 @@ from desk.models import Item
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
RoleCopy, RolePlace, RoleRow, ScoredRole, cv_languages as normal_languages,
language_flag,
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
fetch_terms as normal_terms, host_word, posted_at, posted_host, remote_of, title_key,
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
)
from app.storage import gazetteer, schema
@@ -46,7 +47,9 @@ _RAW_TEXT = ("place", "currency")
_HOST_WORDS = ("seniority", "employment", "remote", "country")
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
#: What prep maps from that a pull rewrites whatever it brings.
_PREP_OVERWRITTEN = ("title", "seniority_host")
#: `deadline_host` is the host's last day to apply (`applicationDeadline` in
#: `raw`, Jobnet and Jobbank), as `YYYY-MM-DD`.
_PREP_OVERWRITTEN = ("title", "seniority_host", "deadline_host")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
@@ -63,13 +66,16 @@ GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: still open, no copy of another
#: (`rebuild_dupes`), not given up, and either never answered, without a
#: model level, or never asked where a remote hire may live
#: (`remote_where_model` NULL). The last two make `extract` the afterburner
#: for the level and the area: a role read before they were asked for is
#: read again, and only fills what is still empty (`set_extracted`). A reply
#: model level, never asked where a remote hire may live
#: (`remote_where_model` NULL), or never asked for its languages and deadline
#: (`languages_required` NULL). The last three make `extract` the
#: afterburner for the level, the area, the languages and the deadline: a
#: role read before they were asked for is read again, and only fills what
#: is still empty (`set_extracted`). A reply
#: that names no level leaves the role waiting and is charged a try
#: (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL) "
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL "
"OR languages_required IS NULL) "
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
@@ -730,8 +736,10 @@ class Store:
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
`remote`, `country`) are left as prep last wrote them. `prep_due`
is 1 on a new role and grows by one when the title, the company, a
host word or the place texts changed or the role reopens, which
is what the next prep reads.
host word, the host's deadline or the place texts changed or the
role reopens, which is what the next prep reads. `deadline_host` is
`raw["applicationDeadline"]` as `YYYY-MM-DD` (`iso_day`), '' when
the pull states none, and prep composes `deadline` from it.
`posted_at` (see `app.models.posted_at`) is set from this pull's own
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
@@ -801,6 +809,7 @@ class Store:
# not kept-filled like _RAW_TEXT: a board that stops stating pay
# must be able to clear it, not leave a stale line behind.
row["salary_text"] = str(raw.get("salary_text") or "")
row["deadline_host"] = iso_day(raw.get("applicationDeadline"))
row["prep_due"] = 1
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
@@ -810,7 +819,7 @@ class Store:
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text", "seniority_host")]
"salary_text", "seniority_host", "deadline_host")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -1421,6 +1430,7 @@ class Store:
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country, remote_model, remote_where_model,
deadline_host, deadline_model, deadline,
json_extract(raw, '$.places') AS raw_places,
(SELECT json_group_array(place_id) FROM (
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
@@ -1446,8 +1456,8 @@ class Store:
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_area` and
`country`. `seniority` is the model's level, else the stated, else
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
the title's. `remote` is written as prep composed it (the model's,
else the host's word, else the place line) and `remote_area`, place
ids, replaces the role's `role_remote`. An empty `country` leaves the
@@ -1464,6 +1474,7 @@ class Store:
employment = :employment,
remote = :remote,
country = COALESCE(NULLIF(:country, ''), country),
deadline = :deadline,
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
WHERE uid = :uid""", mapped).rowcount
conn.executemany("DELETE FROM role_remote WHERE uid = ?",
@@ -1541,7 +1552,8 @@ class Store:
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
.replace("dup_of", "r.dup_of")
.replace("seniority_model", "r.seniority_model")
.replace("remote_where_model", "r.remote_where_model")]
.replace("remote_where_model", "r.remote_where_model")
.replace("languages_required", "r.languages_required")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
@@ -1595,7 +1607,10 @@ class Store:
`remote_model` and `remote_where_model` (a JSON list, `[]` for none),
even when empty, since an answer is what the model read this time;
`remote` and the area are composed from them by prep, so the role is
marked due (`prep.run_roles`, which `extract` calls)."""
marked due (`prep.run_roles`, which `extract` calls).
`languages_required` (a JSON list, `[]` for none) and the deadline,
as `deadline_model` (`YYYY-MM-DD` or ''), are written outright too;
prep composes `deadline` from the host's and this one."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1620,9 +1635,14 @@ class Store:
sets["languages"] = json.dumps(languages)
remote = str(fields.get("remote") or "")
where = [str(name) for name in fields.get("remote_where") or []]
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where)
named = list(fields.get("languages_required") or [])
deadline = str(fields.get("deadline") or "")
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where) \
+ bool(named) + bool(deadline)
sets["remote_model"] = remote
sets["remote_where_model"] = json.dumps(where)
sets["languages_required"] = json.dumps(named)
sets["deadline_model"] = deadline
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
@@ -1880,15 +1900,22 @@ class Store:
weights: Weights | None = None, enabled: bool = True,
agent_id: int | None = None, limits: Limits | None = None,
prompts: Sequence[Prompt] | None = None,
fetch_terms: Sequence[str] | None = None) -> Agent:
fetch_terms: Sequence[str] | None = None,
cv_text: str | None = None,
cv_languages: Sequence[CvLanguage] | None = None,
avoid_text: str | None = None,
must_text: str | None = None) -> Agent:
"""Create an agent, or with `agent_id` replace that one's fields and
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
named after the agent. Without `fetch_terms` an update keeps the
stored ones and a new agent has none, so a caller that does not know
them (the CLI's prompt edit) loses none. A prompt whose text is unchanged keeps its
cached vector, whatever its name or place, so an edit re-embeds only
the prompts it changed. Raises ValueError on an empty or taken name,
or two prompts of one name, KeyError on an unknown id."""
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
`avoid_text` and `must_text` are kept the same way. A prompt whose text is
unchanged keeps its cached vector, whatever its name or place, so an
edit re-embeds only the prompts it changed. A save gives every role
`judge` gave up on for the agent its tries back, since what it is
judged against may have changed. Raises ValueError on an empty or
taken name, or two prompts of one name, KeyError on an unknown id."""
name = name.strip()
if not name:
raise ValueError("an agent needs a name")
@@ -1897,23 +1924,36 @@ class Store:
prompts = check_prompts(list(prompts))
terms = None if fetch_terms is None else json.dumps(
normal_terms(list(fetch_terms)), ensure_ascii=False)
spoken = None if cv_languages is None else json.dumps(
[lang.model_dump() for lang in normal_languages(list(cv_languages))])
fields = (name, json.dumps([g.model_dump() for g in gates]),
(weights or Weights()).model_dump_json(),
(limits or Limits()).model_dump_json(), int(enabled), terms)
(limits or Limits()).model_dump_json(), int(enabled), terms,
None if cv_text is None else cv_text.strip(), spoken,
None if avoid_text is None else avoid_text.strip(),
None if must_text is None else must_text.strip())
try:
with self.transaction() as conn:
if agent_id is None:
agent_id = conn.execute(
"""INSERT INTO agents (name, gates, weights, limits, enabled,
fetch_terms, created_at)
VALUES (?,?,?,?,?,COALESCE(?, '[]'),?)""",
fetch_terms, cv_text, cv_languages, avoid_text,
must_text, created_at)
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),?)""",
(*fields, self.now())).lastrowid
elif not conn.execute(
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
fetch_terms=COALESCE(?, fetch_terms)
fetch_terms=COALESCE(?, fetch_terms),
cv_text=COALESCE(?, cv_text),
cv_languages=COALESCE(?, cv_languages),
avoid_text=COALESCE(?, avoid_text),
must_text=COALESCE(?, must_text)
WHERE id=?""",
(*fields, agent_id)).rowcount:
raise KeyError(agent_id)
conn.execute("UPDATE role_judgements SET tries = 0 "
"WHERE agent_id = ? AND tries > 0", (agent_id,))
cached = {r["text"]: (r["vec_key"], r["vec"]) for r in conn.execute(
"SELECT text, vec_key, vec FROM agent_prompts WHERE agent_id=?",
(agent_id,))}