Merge must-have verdicts and the career cap in code (wp-musts)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-10-03 22:41:26 +02:00
commit 82afa831f2
19 files changed
+336 -61

No files matched your search

+65 -11
View File
@@ -6,7 +6,10 @@ written profile and scores skills, experience and career 0 to 100, with one
to three strengths and gaps. The total is `app.models.cv_score`, the three
numbers under the agent's `cv_*` weights, worked out when a role is served and
not stored, so the weights are no part of `input_key`; it is what the feed's
Best CV match sorts by. Culture and location are left out: the first
Best CV match sorts by. The model also reports a verdict on each of the
agent's must-haves (one per line of its must text, numbered M1..Mn in the
prompt); it does not apply them. The rule is code's: a "broken" one makes
career count as at most `app.models.BROKEN_CAP`, when the role is served. Culture and location are left out: the first
is guessed from ad text, the second is what the gates do.
The work is pairs, `(agent, role)`: an enabled agent with a CV, and a role
@@ -44,13 +47,20 @@ from pipeline import NULL, RunContext, Task, job
from pipeline.task import Batch
from app.config import Config
from app.jobs import stages
from app.models import LANGUAGE_LEVELS, Agent, CvLanguage, cv_languages
from app.models import (
LANGUAGE_LEVELS, MUST_VERDICTS, Agent, CvLanguage, cv_languages, must_lines,
)
from app.storage import Store
#: How much of a posting goes in one item: summarise's 8000, since the
#: judgement needs the whole role, not its opening and closing lines.
EXCERPT = 8000
#: The format of what the judge asks and stores, part of `input_key`: a
#: change to the prompt's contract makes every judgement be made again.
#: 2: the verdict on each numbered must-have (`musts`), the cap in code.
FORMAT = 2
#: How many strengths and gaps are kept, and how long each may be.
MOST, LONGEST = 3, 300
@@ -73,9 +83,17 @@ Score three dimensions, each 0 to 100:
aligned with their direction, with clear growth; 60-79 a good role, partly
aligned; 40-59 a decent job that does not build toward their goals; 0-39 a
dead end or a step backwards. Judge whether the tasks would energise them,
not only whether they could do them. A posting that breaks one of their
stated deal-breakers scores career 0-39, and the deal-breaker is named as a
gap.
not only whether they could do them.
Their must-haves and deal-breakers are numbered M1, M2 and so on. Give a
verdict on every one of them, in `musts`, for each item:
- "met": the posting text says it is met.
- "broken": only when the posting text clearly contradicts it.
- "unclear": the text does not say. Never infer from silence: a posting that
is silent on a must-have is "unclear", not "broken".
Judge a must-have by its own words, not by how well the role fits otherwise.
Answer musts as [] when there are none. The verdicts do not change the three
scores, which follow their rubrics; name a broken must-have as a gap.
Then 1 to 3 strengths and 1 to 3 gaps, each one short sentence grounded in
the posting text. State the gaps honestly: a prestigious role that fits
@@ -100,8 +118,21 @@ SCHEMA: dict[str, Any] = {
"career": {"type": "integer"},
"strengths": {"type": "array", "items": {"type": "string"}},
"gaps": {"type": "array", "items": {"type": "string"}},
"musts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {"type": "string"},
"verdict": {"type": "string", "enum": list(MUST_VERDICTS)},
},
"required": ["id", "verdict"],
"additionalProperties": False,
},
},
},
"required": ["key", "skills", "experience", "career", "strengths", "gaps"],
"required": ["key", "skills", "experience", "career", "strengths", "gaps",
"musts"],
"additionalProperties": False,
},
},
@@ -144,7 +175,7 @@ def input_key(agent: Agent, text: str) -> str:
"""A hash of everything a judgement reads: the CV, the prompts, the
avoid and must texts, and the role's text as sent. A judgement stored
under another key is read again."""
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
blob = json.dumps([FORMAT, agent.cv_text, [[p.name, p.text] for p in agent.prompts],
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
return hashlib.sha1(blob.encode()).hexdigest()[:16]
@@ -199,9 +230,9 @@ def system(agent: Agent) -> str:
if agent.avoid_text.strip():
parts.append("\n<avoid title=\"what drains them and environments to avoid\">\n"
f"{agent.avoid_text.strip()}\n</avoid>")
if agent.must_text.strip():
parts.append("\n<must title=\"must-haves and deal-breakers\">\n"
f"{agent.must_text.strip()}\n</must>")
if musts := must_lines(agent.must_text):
lines = "\n".join(f"M{n}: {line}" for n, line in enumerate(musts, start=1))
parts.append(f"\n<must title=\"must-haves and deal-breakers\">\n{lines}\n</must>")
return "\n".join(parts)
@@ -243,6 +274,25 @@ def _lines(value: Any) -> list[str]:
return [line for line in found if line][:MOST]
def _musts(agent: Agent | None, value: Any) -> list[dict[str, str]]:
"""The verdicts on the agent's must-haves as they are stored: one
`{id, text, verdict}` per line of its must text, in order. A must-have
the model left out, or gave a verdict outside "met", "broken", "unclear",
is "unclear": nothing is broken by silence. An id that is not one of the
agent's is ignored, the first answer for an id counts."""
said: dict[str, str] = {}
for one in value if isinstance(value, list) else []:
if isinstance(one, dict):
said.setdefault(str(one.get("id") or "").strip().upper(),
str(one.get("verdict") or "").strip().lower())
found = []
for n, line in enumerate(must_lines(agent.must_text if agent else ""), start=1):
verdict = said.get(f"M{n}", "")
found.append({"id": f"M{n}", "text": line,
"verdict": verdict if verdict in MUST_VERDICTS else "unclear"})
return found
def apply_batch(store: Store, answer: dict[str, Any],
batch: dict[str, Any] | None = None,
config: Config | None = None) -> JudgeResult:
@@ -251,6 +301,7 @@ def apply_batch(store: Store, answer: dict[str, Any],
all three numbers; the pair stays waiting."""
keys: dict[str, str] = dict((batch or {}).get("keys") or {})
stats = JudgeResult()
agents: dict[int, Agent | None] = {}
for one in answer.get("judgements") or []:
key = str(one.get("key") or "")
pair = pair_of(key)
@@ -259,8 +310,11 @@ def apply_batch(store: Store, answer: dict[str, Any],
stats.rejected.append(key or "?")
continue
skills, experience, career = (int(m) for m in marks if m is not None)
if pair[0] not in agents:
agents[pair[0]] = store.agent(pair[0])
if not store.set_judgement(pair[0], pair[1], keys.pop(key), skills, experience, career,
_lines(one.get("strengths")), _lines(one.get("gaps"))):
_lines(one.get("strengths")), _lines(one.get("gaps")),
_musts(agents[pair[0]], one.get("musts"))):
stats.rejected.append(key)
continue
stats.judged += 1
+53 -7
View File
@@ -1007,30 +1007,76 @@ def language_flag(required: list[RoleLanguage],
return "", ""
#: What career counts as at most when a posting breaks one of the agent's
#: must-haves: the top of the rubric's 0-39 band for a dead end. The model
#: reports whether each must-have is met; this cap is applied in code, when a
#: role is served (`cv_score`, `Store.judgement`, `store._cv_sql`).
BROKEN_CAP = 39
#: A must-have's verdict, as the judge answers it.
MUST_VERDICTS = ("met", "broken", "unclear")
_BULLET = re.compile(r"^\s*(?:[-*\u2022]+|\(?\d+[.)])\s+")
def must_lines(text: str) -> list[str]:
"""The agent's must-haves text read as one must-have per line: a leading
bullet (`-`, `*`, `•`) or number (`1.`, `2)`) is dropped, so are empty
lines."""
found = []
for line in text.splitlines():
line = " ".join(_BULLET.sub("", line, count=1).split())
if line:
found.append(line)
return found
def capped_career(career: int, broken: int = 0) -> int:
"""`career` as it counts: at most `BROKEN_CAP` when any must-have is
broken."""
return min(career, BROKEN_CAP) if broken else career
def cv_score(skills: int, experience: int, career: int,
weights: "Weights | None" = None) -> float | None:
weights: "Weights | None" = None, broken: int = 0) -> float | None:
"""The one number of a CV match, 0 to 100: the three numbers of the
judge averaged by the agent's `cv_skills`, `cv_experience` and
`cv_career` weights (30, 25 and 30 by default). None when all three
weights are 0."""
`cv_career` weights (30, 25 and 30 by default), career as
`capped_career` has it. None when all three weights are 0."""
w = weights or Weights()
total = w.cv_skills + w.cv_experience + w.cv_career
if not total:
return None
career = capped_career(career, broken)
return round((w.cv_skills * skills + w.cv_experience * experience
+ w.cv_career * career) / total, 1)
class Must(Model):
"""One must-have of an agent as the judge read it against a role: its
`id` (`M1`, by line), the line's `text` and the `verdict`."""
id: str
text: str = ""
verdict: Literal["met", "broken", "unclear"] = "unclear"
class Judgement(Model):
"""What `judge` read of one role for one agent (`role_judgements`): the
three dimensions 0 to 100, `cv_score` (`cv_score`) and 1 to 3 strengths
and gaps grounded in the posting. `cv_score` is not read from the table:
`Store.judgement` sets it from the agent's weights; None when all three
are 0."""
three dimensions 0 to 100 as the model gave them, `cv_score`
(`cv_score`) and 1 to 3 strengths and gaps grounded in the posting.
`musts` is the verdict on each must-have and `broken` how many are
broken; any makes `career_capped` the lower of `career` and
`BROKEN_CAP`. `cv_score` and `career_capped` are not read from the
table: `Store.judgement` sets them from the agent's weights; `cv_score`
is None when all three are 0."""
skills: int
experience: int
career: int
career_capped: int | None = None
broken: int = 0
musts: Annotated[list[Must], BeforeValidator(lambda v: _json(v, "[]"))] = []
cv_score: float | None = None
strengths: JsonStrings = []
gaps: JsonStrings = []
+3
View File
@@ -21,6 +21,9 @@
* The CV fieldset holds what the judge stage reads a role against: the CV as
* text (typed, or read from a .md/.txt file by the browser), the languages it
* states as rows of a code and a level, and the avoid and must-have texts.
* The must-have text is one must-have per line, which the judge answers one
* verdict each; a broken one caps career at 39 in code (the hint in
* settings/agents.html says so).
* Every save sends all four, and the card's switch re-posts the row as the
* server gave it, so neither drops them.
*
+8
View File
@@ -928,6 +928,14 @@ body[data-reader="open"] .company-strip { display: none; }
.reader-cv .cv-total { margin: 0 0 .45rem; font-variant-numeric: tabular-nums; }
.reader-cv .cv-total b { font-size: 1.25rem; }
.reader-cv .cv-total span { display: inline-block; margin-left: .4rem; color: var(--dim); font-size: .88rem; }
/* The must-haves, one line each with its verdict's mark. */
.cv-musts { list-style: none; margin: 0 0 .55rem; padding: 0; font-size: .92rem; line-height: 1.4; }
.cv-musts li { display: flex; gap: .45rem; }
.cv-musts li + li { margin-top: .15rem; }
.cv-musts .mark { flex: none; width: 1rem; text-align: center; font-weight: 700; }
.cv-musts .must-met .mark { color: var(--good); }
.cv-musts .must-broken .mark { color: var(--error); }
.cv-musts .must-unclear .mark { color: var(--faint); }
.cv-lists { display: grid; grid-template-columns: 1fr 1fr; gap: .5rem 1.2rem; }
.cv-lists ul { margin: 0; padding-left: 1.1rem; font-size: .92rem; line-height: 1.45; }
.cv-lists li + li { margin-top: .2rem; }
+16 -2
View File
@@ -1412,8 +1412,12 @@
try { return languageNames?.of(code) || code; } catch (err) { return code; }
}
const MUST_MARKS = { met: ["✓", "met"], broken: ["✗", "broken"], unclear: ["?", "unclear"] };
/* The agent's `judge` reading against its CV: the total and its three
numbers, then the strengths and gaps it named. A box as the summary is,
numbers, career as the server counted it (capped at 39 when a must-have
is broken, with what the model said), the must-haves with a verdict
each, then the strengths and gaps it named. A box as the summary is,
since it is the same kind of thing: a model's reading, not the posting. */
function cvMatch(r) {
const j = r.judgement;
@@ -1421,10 +1425,20 @@
const list = (items, none) => (items?.length
? `<ul>${items.map((x) => `<li>${esc(x)}</li>`).join("")}</ul>`
: `<p class="muted small">${none}</p>`);
const counted = j.career_capped ?? j.career;
const career = counted < j.career
? `career ${counted} (capped: breaks a must-have; model said ${j.career})` : `career ${j.career}`;
const musts = Array.isArray(j.musts) && j.musts.length
? `<ul class="cv-musts">${j.musts.map((m) => {
const [mark, word] = MUST_MARKS[m.verdict] || MUST_MARKS.unclear;
return `<li class="must-${word}"><span class="mark" role="img"
aria-label="${word}" title="${word}">${mark}</span><span>${esc(m.text)}</span></li>`;
}).join("")}</ul>` : "";
return `<section class="reader-summary reader-cv">
<h2 class="reader-label">CV match</h2>
<p class="cv-total"><b>${Math.round(j.cv_score)}</b> of 100
<span>skills ${j.skills} · experience ${j.experience} · career ${j.career}</span></p>
<span>skills ${j.skills} · experience ${j.experience} · ${career}</span></p>
${musts}
<div class="cv-lists">
<div><h3 class="reader-label">Strengths</h3>${list(j.strengths, "None named.")}</div>
<div><h3 class="reader-label">Gaps</h3>${list(j.gaps, "None named.")}</div>
+24 -2
View File
@@ -400,8 +400,11 @@ CREATE TABLE IF NOT EXISTS scores (
-- and the role closing. skills, experience and career are 0 to 100,
-- cv_score their weighted total, strengths and gaps JSON lists. input_key
-- is a hash of the CV, prompts, avoid text and the role's text it read: a
-- different one is read again. tries counts replies that left the role
-- without one since; at 3 it is given up until the agent is saved again. A
-- different one is read again. musts is a JSON list of {id, text, verdict},
-- the model's verdict on each of the agent's must-haves (one per line of its
-- must text), broken how many are "broken": career counts as at most 39 then
-- (app.models.BROKEN_CAP), applied when the role is served. tries counts
-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A
-- row with judged_at NULL holds only tries.
CREATE TABLE IF NOT EXISTS role_judgements (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
@@ -412,6 +415,8 @@ CREATE TABLE IF NOT EXISTS role_judgements (
cv_score REAL,
strengths TEXT NOT NULL DEFAULT '[]',
gaps TEXT NOT NULL DEFAULT '[]',
musts TEXT NOT NULL DEFAULT '[]',
broken INTEGER NOT NULL DEFAULT 0,
input_key TEXT NOT NULL DEFAULT '',
tries INTEGER NOT NULL DEFAULT 0,
judged_at TEXT,
@@ -698,6 +703,10 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
(28, "agents.cv_text, cv_languages, avoid_text and must_text, roles.languages_required and the "
"deadline columns, and role_judgements: the CV match, the language flag and the "
"deadline", lambda conn: _add_cv_match(conn)),
# A Python step: it adds the columns only where they are missing. Rows
# keep their numbers; the judge reads them again (judge.FORMAT).
(29, "role_judgements.musts and broken: the verdict on each must-have, and how many are "
"broken", lambda conn: _add_musts(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -1025,6 +1034,19 @@ def _add_cv_match(conn: sqlite3.Connection) -> None:
PRIMARY KEY (uid, agent_id))""")
def _add_musts(conn: sqlite3.Connection) -> None:
"""Migration 29's step: `role_judgements.musts`, a JSON list, and
`broken`, the count of must-haves it holds as broken, empty and 0 on
every row. `judge.input_key` changed with it, so each judgement is read
again; until then it serves with no must-haves and no cap. Runs twice
without harm."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
for column, ddl in (("musts", "TEXT NOT NULL DEFAULT '[]'"),
("broken", "INTEGER NOT NULL DEFAULT 0")):
if column not in columns:
conn.execute(f"ALTER TABLE role_judgements ADD COLUMN {column} {ddl}")
def _add_all_roles(conn: sqlite3.Connection) -> None:
"""Migration 21's step. The column; the tag for every company that is a
favourite of any agent, since favourite no longer widens what a pull
+29 -16
View File
@@ -11,7 +11,7 @@ from __future__ import annotations
import json
import sqlite3
from collections import Counter
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence
from contextlib import closing, contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -26,7 +26,7 @@ from app.models import (
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
RoleCopy, RolePlace, RoleRow, ScoredRole, cv_languages as normal_languages, cv_score,
language_flag, role_languages,
BROKEN_CAP, capped_career, language_flag, role_languages,
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
)
@@ -120,7 +120,7 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
#: The agent's `judge` readings beside every role, the same way: `j.skills`,
#: `j.experience` and `j.career` are NULL for a role not judged and with no
#: `j.experience`, `j.career` and `j.broken` are NULL for a role not judged and with no
#: agent.
_WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? "
"AND j.judged_at IS NOT NULL")
@@ -129,15 +129,19 @@ _WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agen
def _cv_sql(weights: Weights | None) -> str:
"""`cv_score` as SQL for one agent's weights, unrounded (`app.models.cv_score`
rounds it): NULL for a role not judged, and for all three weights 0. The
rounds it): NULL for a role not judged, and for all three weights 0. Career
counts as at most `BROKEN_CAP` when the judgement holds a broken
must-have (`j.broken`), as `app.models.capped_career` has it. The
numbers are the model's validated, finite floats, so they are written in
rather than bound, the select's placeholders being then no concern."""
w = weights or Weights()
total = w.cv_skills + w.cv_experience + w.cv_career
if not total:
return "NULL"
career = (f"(CASE WHEN j.broken > 0 AND j.career > {BROKEN_CAP} THEN {BROKEN_CAP} "
"ELSE j.career END)")
return (f"(({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience "
f"+ {w.cv_career!r} * j.career) / {total!r})")
f"+ {w.cv_career!r} * {career}) / {total!r})")
def _by_cv(expression: str) -> str:
@@ -1888,27 +1892,34 @@ class Store:
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
experience: int, career: int, strengths: Sequence[str],
gaps: Sequence[str]) -> bool:
gaps: Sequence[str], musts: Sequence[Mapping[str, str]] = ()) -> bool:
"""Store `judge`'s reading of one role for one agent, replacing the
one before, with `cv_score` (`app.models.cv_score`, at the default weights: the
table's column is not read, see `judgement`) and its tries
back at 0. False for an unknown role or agent."""
one before: the three numbers as the model gave them, the verdict on
each of the agent's must-haves (`musts`, `{id, text, verdict}`) and how
many are broken, `cv_score` (`app.models.cv_score`, at the default
weights and with the cap: the table's column is not read, see
`judgement`) and its tries back at 0. False for an unknown role or
agent."""
broken = sum(1 for m in musts if m.get("verdict") == "broken")
try:
with self.transaction() as conn:
conn.execute(
"""INSERT INTO role_judgements (uid, agent_id, skills, experience, career,
cv_score, strengths, gaps, input_key, tries,
judged_at)
VALUES (?,?,?,?,?,?,?,?,?,0,?)
cv_score, strengths, gaps, musts, broken,
input_key, tries, judged_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,0,?)
ON CONFLICT(uid, agent_id) DO UPDATE SET
skills=excluded.skills, experience=excluded.experience,
career=excluded.career, cv_score=excluded.cv_score,
strengths=excluded.strengths, gaps=excluded.gaps,
musts=excluded.musts, broken=excluded.broken,
input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""",
(uid, agent_id, skills, experience, career,
cv_score(skills, experience, career),
cv_score(skills, experience, career, broken=broken),
json.dumps(list(strengths), ensure_ascii=False),
json.dumps(list(gaps), ensure_ascii=False), input_key, self.now()))
json.dumps(list(gaps), ensure_ascii=False),
json.dumps([dict(m) for m in musts], ensure_ascii=False), broken,
input_key, self.now()))
except sqlite3.IntegrityError:
return False
return True
@@ -1935,7 +1946,8 @@ class Store:
def judgement(self, agent_id: int | None, uid: str) -> Judgement | None:
"""The agent's judgement of one role, None when it has none, its
`cv_score` the three numbers under the agent's weights as they are
now, not as stored."""
now, not as stored, and `career_capped` the model's career as the
broken must-haves leave it (`BROKEN_CAP`)."""
if agent_id is None:
return None
row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? "
@@ -1944,8 +1956,9 @@ class Store:
return None
found = Judgement.from_row(row)
agent = self.agent(agent_id)
found.career_capped = capped_career(found.career, found.broken)
found.cv_score = cv_score(found.skills, found.experience, found.career,
agent.weights if agent else None)
agent.weights if agent else None, found.broken)
return found
def set_article_text(self, uid: str, text: str) -> Role | None:
+2 -2
View File
@@ -96,8 +96,8 @@
<label>Avoid / drains me
<textarea id="a-avoid-text" rows="4" placeholder="Constant on-call, sales targets, large consultancies, open-plan offices"></textarea></label>
<label>Must-haves and deal-breakers
<textarea id="a-must-text" rows="4" placeholder="Permanent contract, a say in the roadmap, at most one day a week in the office"></textarea></label>
<span class="hint">Soft, read by the judge only: a role that breaks one of these scores low on career and the gap is named. Rules that must hold go under Where or Other rules instead.</span>
<textarea id="a-must-text" rows="4" placeholder="Permanent contract&#10;A say in the roadmap&#10;At most one day a week in the office"></textarea></label>
<span class="hint">One must-have per line. The judge reports for each whether a posting meets it, clearly breaks it or does not say; a broken one caps the role's career score at 39 in code, whatever the model scored. Rules that must hold go under Where or Other rules instead.</span>
</fieldset>
<fieldset class="weights">
<legend>How much each counts</legend>
+1 -1
View File
@@ -208,7 +208,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**`extract_batch` and `summarise_batch` are `isolate=True`, with a timeout.** `context.check()` only runs between the calls `modeltask.one` makes, not inside one, so a call that hangs on either backend was otherwise unkillable short of restarting the server; a process boundary is what lets a Kill, or the timeout itself, actually end it. One call is already bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for summarise): a batch still running after that is stuck, not slow. The cost: an isolated job's context has no working `stop`, `task` or `fan_out` (see `pipeline/child.py`'s `context_for`), so a refusal or an account-level failure inside a batch can no longer stop the rest of that task's queued siblings, which run on to the same failure instead of being spared it.
**A third order, Best CV match, from a model reading each agent's best roles against its CV (2026-10-03).** Taken from the `/rank` of github.com/MadsLorentzen/ai-job-search, which has a model score a posting against a written profile on five dimensions. Fit compares topics; it cannot tell a step down, years of experience or a stated deal-breaker, which a model reading both texts can. An agent may hold a CV (pasted, or a `.md`/`.txt` file the browser reads into the box), its languages with a level (suggested from the CV by one model call, then edited), an "avoid / drains me" text and a "must-haves and deal-breakers" text; the prompts stay what the owner wants. The stage `judge` (after summarise in the pull, switched under Runs like extract, Haiku by default) reads the pairs of an enabled agent with a CV within its `judge_top` and `judge_min` limits, once per `input_key` (a hash of the CV, prompts, both texts and the role's text), and stores skills, experience and career (0 to 100, their rubric bands verbatim), 1 to 3 strengths and gaps in `role_judgements`, which rescoring and closing never touch. The CV match is the weighted mean of the three, with the weights per agent (`cv_skills`, `cv_experience`, `cv_career` in `agents.weights`, default 30, 25, 30) applied when a role is served, so changing them needs no model call. Culture is left out (guessed from ad text) and location too (the gates do it): a must-haves text that names a place makes the model mark location down, so places belong in the Where rules. Haiku does not always hold the deal-breaker band (0 to 39 for career); a stronger model or a clamp in code is the fix if that matters. Their scale is ten postings a run; ours is the top of a ranking that has already scored every role for free.
**A third order, Best CV match, from a model reading each agent's best roles against its CV (2026-10-03).** Taken from the `/rank` of github.com/MadsLorentzen/ai-job-search, which has a model score a posting against a written profile on five dimensions. Fit compares topics; it cannot tell a step down, years of experience or a stated deal-breaker, which a model reading both texts can. An agent may hold a CV (pasted, or a `.md`/`.txt` file the browser reads into the box), its languages with a level (suggested from the CV by one model call, then edited), an "avoid / drains me" text and a "must-haves and deal-breakers" text; the prompts stay what the owner wants. The stage `judge` (after summarise in the pull, switched under Runs like extract, Haiku by default) reads the pairs of an enabled agent with a CV within its `judge_top` and `judge_min` limits, once per `input_key` (a hash of the CV, prompts, both texts and the role's text), and stores skills, experience and career (0 to 100, their rubric bands verbatim), 1 to 3 strengths and gaps in `role_judgements`, which rescoring and closing never touch. The CV match is the weighted mean of the three, with the weights per agent (`cv_skills`, `cv_experience`, `cv_career` in `agents.weights`, default 30, 25, 30) applied when a role is served, so changing them needs no model call. Culture is left out (guessed from ad text) and location too (the gates do it): a must-haves text that names a place makes the model mark location down, so places belong in the Where rules. The model reports facts and code applies the rules: the must-haves text is read as one must-have per line (leading bullets and numbers dropped), the judge's profile numbers them M1..Mn, and for each judgement the model answers `musts`, a verdict per must-have, "met", "broken" only when the posting text clearly contradicts it, or "unclear" when the text does not say (never inferred from silence; a must-have the model leaves out counts as unclear). It scores career on its rubric as before and the career is stored as given. The cap is code's: when a role is served, any "broken" verdict makes career count as `min(career, BROKEN_CAP)` with `BROKEN_CAP = 39` (`app.models`), in `cv_score`, in `Store.judgement` (which serves the model's career, `career_capped` and the must-haves with their text) and in `store._cv_sql`, which reads a `broken` count stored beside the verdicts (`role_judgements.musts` and `.broken`, migration 29), so Best CV match orders by the capped value too. The verdicts store each line's text, so the reader still shows the must-haves a judgement was made against after the text is edited. The judge's `input_key` carries a format version (`judge.FORMAT`, 2), so judgements made before the verdicts existed are made again; until then they serve with no must-haves and no cap. The reader shows each must-have with ✓ met, ✗ broken or ? unclear, and career as "39 (capped: breaks a must-have; model said 55)". Their scale is ten postings a run; ours is the top of a ranking that has already scored every role for free.
**A posting's languages and deadline, and a flag rather than a score.** Extract reads `languages_required` as objects (language, level on the ladder basic, conversational, professional, fluent, native, and required or plus) and a deadline, from the opening and the closing lines of the text, since both usually sit at the end. Jobnet and Jobbank state a deadline, kept as `deadline_host`; prep composes `deadline` host first. The language flag is computed when a role is served from the posting's languages and the agent's: ⚠ when a required one is missing or its level is above the agent's, ★ when a plus one is the agent's; it never changes a score. Closing soon is the deadline from today to today plus four days, in UTC.
+2 -2
View File
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28]
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28]
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29]
# The tag of 21 is All countries and All roles since 24.
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
+31
View File
@@ -91,6 +91,37 @@ def test_best_cv_match_puts_the_judged_first_by_cv_score(
"order": "best"}).status_code == 422
def test_a_broken_must_have_caps_career_in_the_order_and_the_reader(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
# c: 90 on all three, but one must-have broken: career counts as 39.
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
{"id": "M2", "text": "Remote", "verdict": "broken"}]
store.set_judgement(owner.id, uid["c"], "k", 90, 90, 90, [], [], musts)
capped = round((30 * 90 + 25 * 90 + 30 * 39) / 85, 1)
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["c"], capped), (uid["b"], 60.0)]
# With career weighted up the capped value drops under b's 60.
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=owner.cv_text,
weights=Weights(cv_skills=0, cv_experience=0, cv_career=1))
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["b"], 60.0), (uid["c"], 39.0)]
one = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()
j = one["judgement"]
assert (j["career"], j["career_capped"], j["broken"]) == (90, 39, 1)
assert j["musts"] == musts and one["cv_score"] == j["cv_score"] == 39.0
def test_a_met_or_unclear_must_have_leaves_career_as_the_model_gave_it(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
{"id": "M2", "text": "Remote", "verdict": "unclear"}]
store.set_judgement(owner.id, uid["c"], "k", 90, 90, 90, [], [], musts)
j = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()["judgement"]
assert (j["career"], j["career_capped"], j["broken"], j["cv_score"]) == (90, 90, 0, 90.0)
def test_the_agents_weights_make_the_cv_score_at_read_time(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
+25 -3
View File
@@ -16,7 +16,7 @@ from pydantic import ValidationError
from app import prep
from app.config import Config
from app.jobs import extract
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag, must_lines
from app.storage import Store
from desk import modeltask
from desk.models import Item
@@ -93,7 +93,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
migrated = Store(path)
assert migrated.migrations_applied == [28]
assert migrated.migrations_applied == [28, 29]
rows = {r["source_id"]: r for r in migrated._rows(
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
@@ -105,7 +105,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 27")
assert Store(path).migrations_applied == [28]
assert Store(path).migrations_applied == [28, 29]
# ---------------------------------------------------------------- deadline
@@ -236,3 +236,25 @@ def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
(json.dumps([{"language": "da", "level": "fluent", "kind": "required"},
{"language": "en", "level": "superb"}]), item.uid))
assert [lang.language for lang in store.role(item.uid).languages_required] == ["da"]
def test_v29_adds_the_musts_and_broken_columns_and_runs_twice(tmp_path: Path) -> None:
path = tmp_path / "v28.db"
Store(path)
with sqlite3.connect(path) as conn:
for column in ("musts", "broken"):
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
conn.execute("PRAGMA user_version = 28")
assert Store(path).migrations_applied == [29]
with sqlite3.connect(path) as conn:
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
assert {"musts", "broken"} <= columns
conn.execute("PRAGMA user_version = 28")
assert Store(path).migrations_applied == [29]
def test_the_musts_text_is_one_must_have_per_line() -> None:
assert must_lines("- A\n* B c\n\u2022 D\n1. E\n2) F\n \nG\n-H") == [
"A", "B c", "D", "E", "F", "G", "-H"]
assert must_lines("") == []
+1 -1
View File
@@ -261,7 +261,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+1 -1
View File
@@ -434,7 +434,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
conn.execute("PRAGMA user_version = 22")
migrated = Store(path)
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29]
assert migrated.agents()[0].fetch_terms == []
+60
View File
@@ -4,6 +4,8 @@ the stage. Nothing here reaches a model."""
from __future__ import annotations
import hashlib
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -109,6 +111,64 @@ def test_a_run_writes_the_judgement_and_its_total(
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
[uid] = ranked(store, owner, "1")
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [{**one, "career": 80, "musts": [
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
for one in judged(items)]
model = stub(monkeypatch, reply)
modeltask.run(judge.TASK, config, store)
[system] = model.systems
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
found = store.judgement(owner.id, uid)
assert found is not None
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
("M3", "Danish team", "unclear")]
assert (found.career, found.career_capped, found.broken) == (80, 39, 1)
assert found.cv_score == round((30 * 80 + 25 * 70 + 30 * 39) / 85, 1)
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch) # answers without any `musts`
modeltask.run(judge.TASK, config, store)
found = store.judgement(owner.id, uid)
assert found is not None
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
assert (found.career, found.career_capped, found.broken) == (90, 90, 0)
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
item = judge.SCHEMA["properties"]["judgements"]["items"]
assert "musts" in item["required"]
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
assert verdict["enum"] == ["met", "broken", "unclear"]
assert "career 0-39" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
def test_judgements_made_before_the_format_are_made_again(store: Store) -> None:
owner = hunter(store)
assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text")
def hashlib_old_key(agent: Any, text: str) -> str:
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
return hashlib.sha1(blob.encode()).hexdigest()[:16]
def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) -> None:
for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})):
ranked(store, hunter(store, name, **kwargs), name)
+1 -1
View File
@@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
conn.execute("PRAGMA user_version = 23")
migrated = Store(path)
assert migrated.migrations_applied == [24, 25, 26, 27, 28]
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29]
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
+1 -1
View File
@@ -316,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
+2 -2
View File
@@ -530,7 +530,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
migrated = Store(path)
assert migrated.migrations_applied == [27, 28]
assert migrated.migrations_applied == [27, 28, 29]
assert migrated.gazetteer().anywhere is not None
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
"WHERE uid = ?", (board,))
@@ -540,7 +540,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
assert area(migrated, search.uid) == ("", "")
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 26")
assert Store(path).migrations_applied == [27, 28]
assert Store(path).migrations_applied == [27, 28, 29]
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
+11 -9
View File
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 28
assert LATEST == 29
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -168,6 +168,8 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
for table in ("vectors", "role_tags", "agent_companies", "role_votes"):
conn.execute(f"DROP TABLE {table}")
for column in ("musts", "broken"): # v29
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
conn.execute("PRAGMA user_version = 1")
assert Store(old).migrations_applied == list(range(2, LATEST + 1))
@@ -187,7 +189,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -392,7 +394,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -416,7 +418,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -445,7 +447,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -688,7 +690,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -730,7 +732,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
migrated = Store(path)
assert migrated.migrations_applied == [25, 26, 27, 28]
assert migrated.migrations_applied == [25, 26, 27, 28, 29]
with sqlite3.connect(path) as conn:
found = {sid: rest for sid, *rest in conn.execute(
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
@@ -748,12 +750,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
migrated = Store(path)
assert migrated.migrations_applied == [26, 27, 28]
assert migrated.migrations_applied == [26, 27, 28, 29]
role = migrated.role(posting("a").uid)
assert role is not None and role.posting_url == ""
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 25") # the column is there already
assert Store(path).migrations_applied == [26, 27, 28]
assert Store(path).migrations_applied == [26, 27, 28, 29]
def test_a_pull_keeps_the_posting_url(store: Store) -> None: