From 001fc9002badc96d469986782762d68a1edbe63e Mon Sep 17 00:00:00 2001 From: Wolf Date: Sat, 3 Oct 2026 22:40:54 +0200 Subject: [PATCH] Let the model report a verdict per must-have and cap career at 39 in code when one is broken The judge numbers an agent's must-haves (one per line) and answers met, broken or unclear for each; career is stored as the model gave it and counts as at most BROKEN_CAP when a role is served, in cv_score, Store.judgement and the SQL of Best CV match. Migration 29 adds role_judgements.musts and broken; the judge's input_key carries a format version so earlier judgements are made again. The reader lists the must-haves with their verdicts and the cap. Co-Authored-By: Claude Sonnet 5.5 --- app/jobs/judge.py | 76 +++++++++++++++++++++++++----- app/models.py | 60 ++++++++++++++++++++--- app/static/agents.js | 3 ++ app/static/app.css | 8 ++++ app/static/roles.js | 18 ++++++- app/storage/schema.py | 26 +++++++++- app/storage/store.py | 45 +++++++++++------- app/templates/settings/agents.html | 4 +- docs/decisions.md | 2 +- tests/test_companies.py | 4 +- tests/test_cv_feed.py | 31 ++++++++++++ tests/test_cv_fields.py | 28 +++++++++-- tests/test_dupes_storage.py | 2 +- tests/test_fetch_terms.py | 2 +- tests/test_judge.py | 60 +++++++++++++++++++++++ tests/test_overrides.py | 2 +- tests/test_prep.py | 2 +- tests/test_remote.py | 4 +- tests/test_storage.py | 20 ++++---- 19 files changed, 336 insertions(+), 61 deletions(-) diff --git a/app/jobs/judge.py b/app/jobs/judge.py index 6b8e253..d04d9ed 100644 --- a/app/jobs/judge.py +++ b/app/jobs/judge.py @@ -6,7 +6,10 @@ written profile and scores skills, experience and career 0 to 100, with one to three strengths and gaps. The total is `app.models.cv_score`, the three numbers under the agent's `cv_*` weights, worked out when a role is served and not stored, so the weights are no part of `input_key`; it is what the feed's -Best CV match sorts by. Culture and location are left out: the first +Best CV match sorts by. The model also reports a verdict on each of the +agent's must-haves (one per line of its must text, numbered M1..Mn in the +prompt); it does not apply them. The rule is code's: a "broken" one makes +career count as at most `app.models.BROKEN_CAP`, when the role is served. Culture and location are left out: the first is guessed from ad text, the second is what the gates do. The work is pairs, `(agent, role)`: an enabled agent with a CV, and a role @@ -44,13 +47,20 @@ from pipeline import NULL, RunContext, Task, job from pipeline.task import Batch from app.config import Config from app.jobs import stages -from app.models import LANGUAGE_LEVELS, Agent, CvLanguage, cv_languages +from app.models import ( + LANGUAGE_LEVELS, MUST_VERDICTS, Agent, CvLanguage, cv_languages, must_lines, +) from app.storage import Store #: How much of a posting goes in one item: summarise's 8000, since the #: judgement needs the whole role, not its opening and closing lines. EXCERPT = 8000 +#: The format of what the judge asks and stores, part of `input_key`: a +#: change to the prompt's contract makes every judgement be made again. +#: 2: the verdict on each numbered must-have (`musts`), the cap in code. +FORMAT = 2 + #: How many strengths and gaps are kept, and how long each may be. MOST, LONGEST = 3, 300 @@ -73,9 +83,17 @@ Score three dimensions, each 0 to 100: aligned with their direction, with clear growth; 60-79 a good role, partly aligned; 40-59 a decent job that does not build toward their goals; 0-39 a dead end or a step backwards. Judge whether the tasks would energise them, - not only whether they could do them. A posting that breaks one of their - stated deal-breakers scores career 0-39, and the deal-breaker is named as a - gap. + not only whether they could do them. + +Their must-haves and deal-breakers are numbered M1, M2 and so on. Give a +verdict on every one of them, in `musts`, for each item: +- "met": the posting text says it is met. +- "broken": only when the posting text clearly contradicts it. +- "unclear": the text does not say. Never infer from silence: a posting that + is silent on a must-have is "unclear", not "broken". +Judge a must-have by its own words, not by how well the role fits otherwise. +Answer musts as [] when there are none. The verdicts do not change the three +scores, which follow their rubrics; name a broken must-have as a gap. Then 1 to 3 strengths and 1 to 3 gaps, each one short sentence grounded in the posting text. State the gaps honestly: a prestigious role that fits @@ -100,8 +118,21 @@ SCHEMA: dict[str, Any] = { "career": {"type": "integer"}, "strengths": {"type": "array", "items": {"type": "string"}}, "gaps": {"type": "array", "items": {"type": "string"}}, + "musts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "verdict": {"type": "string", "enum": list(MUST_VERDICTS)}, + }, + "required": ["id", "verdict"], + "additionalProperties": False, + }, + }, }, - "required": ["key", "skills", "experience", "career", "strengths", "gaps"], + "required": ["key", "skills", "experience", "career", "strengths", "gaps", + "musts"], "additionalProperties": False, }, }, @@ -144,7 +175,7 @@ def input_key(agent: Agent, text: str) -> str: """A hash of everything a judgement reads: the CV, the prompts, the avoid and must texts, and the role's text as sent. A judgement stored under another key is read again.""" - blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts], + blob = json.dumps([FORMAT, agent.cv_text, [[p.name, p.text] for p in agent.prompts], agent.avoid_text, agent.must_text, text], ensure_ascii=False) return hashlib.sha1(blob.encode()).hexdigest()[:16] @@ -199,9 +230,9 @@ def system(agent: Agent) -> str: if agent.avoid_text.strip(): parts.append("\n\n" f"{agent.avoid_text.strip()}\n") - if agent.must_text.strip(): - parts.append("\n\n" - f"{agent.must_text.strip()}\n") + if musts := must_lines(agent.must_text): + lines = "\n".join(f"M{n}: {line}" for n, line in enumerate(musts, start=1)) + parts.append(f"\n\n{lines}\n") return "\n".join(parts) @@ -243,6 +274,25 @@ def _lines(value: Any) -> list[str]: return [line for line in found if line][:MOST] +def _musts(agent: Agent | None, value: Any) -> list[dict[str, str]]: + """The verdicts on the agent's must-haves as they are stored: one + `{id, text, verdict}` per line of its must text, in order. A must-have + the model left out, or gave a verdict outside "met", "broken", "unclear", + is "unclear": nothing is broken by silence. An id that is not one of the + agent's is ignored, the first answer for an id counts.""" + said: dict[str, str] = {} + for one in value if isinstance(value, list) else []: + if isinstance(one, dict): + said.setdefault(str(one.get("id") or "").strip().upper(), + str(one.get("verdict") or "").strip().lower()) + found = [] + for n, line in enumerate(must_lines(agent.must_text if agent else ""), start=1): + verdict = said.get(f"M{n}", "") + found.append({"id": f"M{n}", "text": line, + "verdict": verdict if verdict in MUST_VERDICTS else "unclear"}) + return found + + def apply_batch(store: Store, answer: dict[str, Any], batch: dict[str, Any] | None = None, config: Config | None = None) -> JudgeResult: @@ -251,6 +301,7 @@ def apply_batch(store: Store, answer: dict[str, Any], all three numbers; the pair stays waiting.""" keys: dict[str, str] = dict((batch or {}).get("keys") or {}) stats = JudgeResult() + agents: dict[int, Agent | None] = {} for one in answer.get("judgements") or []: key = str(one.get("key") or "") pair = pair_of(key) @@ -259,8 +310,11 @@ def apply_batch(store: Store, answer: dict[str, Any], stats.rejected.append(key or "?") continue skills, experience, career = (int(m) for m in marks if m is not None) + if pair[0] not in agents: + agents[pair[0]] = store.agent(pair[0]) if not store.set_judgement(pair[0], pair[1], keys.pop(key), skills, experience, career, - _lines(one.get("strengths")), _lines(one.get("gaps"))): + _lines(one.get("strengths")), _lines(one.get("gaps")), + _musts(agents[pair[0]], one.get("musts"))): stats.rejected.append(key) continue stats.judged += 1 diff --git a/app/models.py b/app/models.py index 641768d..d08694e 100644 --- a/app/models.py +++ b/app/models.py @@ -1007,30 +1007,76 @@ def language_flag(required: list[RoleLanguage], return "", "" +#: What career counts as at most when a posting breaks one of the agent's +#: must-haves: the top of the rubric's 0-39 band for a dead end. The model +#: reports whether each must-have is met; this cap is applied in code, when a +#: role is served (`cv_score`, `Store.judgement`, `store._cv_sql`). +BROKEN_CAP = 39 + +#: A must-have's verdict, as the judge answers it. +MUST_VERDICTS = ("met", "broken", "unclear") + +_BULLET = re.compile(r"^\s*(?:[-*\u2022]+|\(?\d+[.)])\s+") + + +def must_lines(text: str) -> list[str]: + """The agent's must-haves text read as one must-have per line: a leading + bullet (`-`, `*`, `•`) or number (`1.`, `2)`) is dropped, so are empty + lines.""" + found = [] + for line in text.splitlines(): + line = " ".join(_BULLET.sub("", line, count=1).split()) + if line: + found.append(line) + return found + + +def capped_career(career: int, broken: int = 0) -> int: + """`career` as it counts: at most `BROKEN_CAP` when any must-have is + broken.""" + return min(career, BROKEN_CAP) if broken else career + + def cv_score(skills: int, experience: int, career: int, - weights: "Weights | None" = None) -> float | None: + weights: "Weights | None" = None, broken: int = 0) -> float | None: """The one number of a CV match, 0 to 100: the three numbers of the judge averaged by the agent's `cv_skills`, `cv_experience` and - `cv_career` weights (30, 25 and 30 by default). None when all three - weights are 0.""" + `cv_career` weights (30, 25 and 30 by default), career as + `capped_career` has it. None when all three weights are 0.""" w = weights or Weights() total = w.cv_skills + w.cv_experience + w.cv_career if not total: return None + career = capped_career(career, broken) return round((w.cv_skills * skills + w.cv_experience * experience + w.cv_career * career) / total, 1) +class Must(Model): + """One must-have of an agent as the judge read it against a role: its + `id` (`M1`, by line), the line's `text` and the `verdict`.""" + + id: str + text: str = "" + verdict: Literal["met", "broken", "unclear"] = "unclear" + + class Judgement(Model): """What `judge` read of one role for one agent (`role_judgements`): the - three dimensions 0 to 100, `cv_score` (`cv_score`) and 1 to 3 strengths - and gaps grounded in the posting. `cv_score` is not read from the table: - `Store.judgement` sets it from the agent's weights; None when all three - are 0.""" + three dimensions 0 to 100 as the model gave them, `cv_score` + (`cv_score`) and 1 to 3 strengths and gaps grounded in the posting. + `musts` is the verdict on each must-have and `broken` how many are + broken; any makes `career_capped` the lower of `career` and + `BROKEN_CAP`. `cv_score` and `career_capped` are not read from the + table: `Store.judgement` sets them from the agent's weights; `cv_score` + is None when all three are 0.""" skills: int experience: int career: int + career_capped: int | None = None + broken: int = 0 + musts: Annotated[list[Must], BeforeValidator(lambda v: _json(v, "[]"))] = [] cv_score: float | None = None strengths: JsonStrings = [] gaps: JsonStrings = [] diff --git a/app/static/agents.js b/app/static/agents.js index a93bb12..2ceca39 100644 --- a/app/static/agents.js +++ b/app/static/agents.js @@ -21,6 +21,9 @@ * The CV fieldset holds what the judge stage reads a role against: the CV as * text (typed, or read from a .md/.txt file by the browser), the languages it * states as rows of a code and a level, and the avoid and must-have texts. + * The must-have text is one must-have per line, which the judge answers one + * verdict each; a broken one caps career at 39 in code (the hint in + * settings/agents.html says so). * Every save sends all four, and the card's switch re-posts the row as the * server gave it, so neither drops them. * diff --git a/app/static/app.css b/app/static/app.css index 1f5455f..3dc5257 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -928,6 +928,14 @@ body[data-reader="open"] .company-strip { display: none; } .reader-cv .cv-total { margin: 0 0 .45rem; font-variant-numeric: tabular-nums; } .reader-cv .cv-total b { font-size: 1.25rem; } .reader-cv .cv-total span { display: inline-block; margin-left: .4rem; color: var(--dim); font-size: .88rem; } +/* The must-haves, one line each with its verdict's mark. */ +.cv-musts { list-style: none; margin: 0 0 .55rem; padding: 0; font-size: .92rem; line-height: 1.4; } +.cv-musts li { display: flex; gap: .45rem; } +.cv-musts li + li { margin-top: .15rem; } +.cv-musts .mark { flex: none; width: 1rem; text-align: center; font-weight: 700; } +.cv-musts .must-met .mark { color: var(--good); } +.cv-musts .must-broken .mark { color: var(--error); } +.cv-musts .must-unclear .mark { color: var(--faint); } .cv-lists { display: grid; grid-template-columns: 1fr 1fr; gap: .5rem 1.2rem; } .cv-lists ul { margin: 0; padding-left: 1.1rem; font-size: .92rem; line-height: 1.45; } .cv-lists li + li { margin-top: .2rem; } diff --git a/app/static/roles.js b/app/static/roles.js index 326ace7..ef884b6 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -1412,8 +1412,12 @@ try { return languageNames?.of(code) || code; } catch (err) { return code; } } + const MUST_MARKS = { met: ["✓", "met"], broken: ["✗", "broken"], unclear: ["?", "unclear"] }; + /* The agent's `judge` reading against its CV: the total and its three - numbers, then the strengths and gaps it named. A box as the summary is, + numbers, career as the server counted it (capped at 39 when a must-have + is broken, with what the model said), the must-haves with a verdict + each, then the strengths and gaps it named. A box as the summary is, since it is the same kind of thing: a model's reading, not the posting. */ function cvMatch(r) { const j = r.judgement; @@ -1421,10 +1425,20 @@ const list = (items, none) => (items?.length ? `` : `

${none}

`); + const counted = j.career_capped ?? j.career; + const career = counted < j.career + ? `career ${counted} (capped: breaks a must-have; model said ${j.career})` : `career ${j.career}`; + const musts = Array.isArray(j.musts) && j.musts.length + ? `` : ""; return `

CV match

${Math.round(j.cv_score)} of 100 - skills ${j.skills} · experience ${j.experience} · career ${j.career}

+ skills ${j.skills} · experience ${j.experience} · ${career}

+ ${musts}

Strengths

${list(j.strengths, "None named.")}

Gaps

${list(j.gaps, "None named.")}
diff --git a/app/storage/schema.py b/app/storage/schema.py index 43110bf..bf8786a 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -400,8 +400,11 @@ CREATE TABLE IF NOT EXISTS scores ( -- and the role closing. skills, experience and career are 0 to 100, -- cv_score their weighted total, strengths and gaps JSON lists. input_key -- is a hash of the CV, prompts, avoid text and the role's text it read: a --- different one is read again. tries counts replies that left the role --- without one since; at 3 it is given up until the agent is saved again. A +-- different one is read again. musts is a JSON list of {id, text, verdict}, +-- the model's verdict on each of the agent's must-haves (one per line of its +-- must text), broken how many are "broken": career counts as at most 39 then +-- (app.models.BROKEN_CAP), applied when the role is served. tries counts +-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A -- row with judged_at NULL holds only tries. CREATE TABLE IF NOT EXISTS role_judgements ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, @@ -412,6 +415,8 @@ CREATE TABLE IF NOT EXISTS role_judgements ( cv_score REAL, strengths TEXT NOT NULL DEFAULT '[]', gaps TEXT NOT NULL DEFAULT '[]', + musts TEXT NOT NULL DEFAULT '[]', + broken INTEGER NOT NULL DEFAULT 0, input_key TEXT NOT NULL DEFAULT '', tries INTEGER NOT NULL DEFAULT 0, judged_at TEXT, @@ -698,6 +703,10 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''), (28, "agents.cv_text, cv_languages, avoid_text and must_text, roles.languages_required and the " "deadline columns, and role_judgements: the CV match, the language flag and the " "deadline", lambda conn: _add_cv_match(conn)), + # A Python step: it adds the columns only where they are missing. Rows + # keep their numbers; the judge reads them again (judge.FORMAT). + (29, "role_judgements.musts and broken: the verdict on each must-have, and how many are " + "broken", lambda conn: _add_musts(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -1025,6 +1034,19 @@ def _add_cv_match(conn: sqlite3.Connection) -> None: PRIMARY KEY (uid, agent_id))""") +def _add_musts(conn: sqlite3.Connection) -> None: + """Migration 29's step: `role_judgements.musts`, a JSON list, and + `broken`, the count of must-haves it holds as broken, empty and 0 on + every row. `judge.input_key` changed with it, so each judgement is read + again; until then it serves with no must-haves and no cap. Runs twice + without harm.""" + columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")} + for column, ddl in (("musts", "TEXT NOT NULL DEFAULT '[]'"), + ("broken", "INTEGER NOT NULL DEFAULT 0")): + if column not in columns: + conn.execute(f"ALTER TABLE role_judgements ADD COLUMN {column} {ddl}") + + def _add_all_roles(conn: sqlite3.Connection) -> None: """Migration 21's step. The column; the tag for every company that is a favourite of any agent, since favourite no longer widens what a pull diff --git a/app/storage/store.py b/app/storage/store.py index 682500d..1c5e5e3 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -11,7 +11,7 @@ from __future__ import annotations import json import sqlite3 from collections import Counter -from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence from contextlib import closing, contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path @@ -26,7 +26,7 @@ from app.models import ( AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role, RoleCopy, RolePlace, RoleRow, ScoredRole, cv_languages as normal_languages, cv_score, - language_flag, role_languages, + BROKEN_CAP, capped_career, language_flag, role_languages, Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key, fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key, ) @@ -120,7 +120,7 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 " _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?" #: The agent's `judge` readings beside every role, the same way: `j.skills`, -#: `j.experience` and `j.career` are NULL for a role not judged and with no +#: `j.experience`, `j.career` and `j.broken` are NULL for a role not judged and with no #: agent. _WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? " "AND j.judged_at IS NOT NULL") @@ -129,15 +129,19 @@ _WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agen def _cv_sql(weights: Weights | None) -> str: """`cv_score` as SQL for one agent's weights, unrounded (`app.models.cv_score` - rounds it): NULL for a role not judged, and for all three weights 0. The + rounds it): NULL for a role not judged, and for all three weights 0. Career + counts as at most `BROKEN_CAP` when the judgement holds a broken + must-have (`j.broken`), as `app.models.capped_career` has it. The numbers are the model's validated, finite floats, so they are written in rather than bound, the select's placeholders being then no concern.""" w = weights or Weights() total = w.cv_skills + w.cv_experience + w.cv_career if not total: return "NULL" + career = (f"(CASE WHEN j.broken > 0 AND j.career > {BROKEN_CAP} THEN {BROKEN_CAP} " + "ELSE j.career END)") return (f"(({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience " - f"+ {w.cv_career!r} * j.career) / {total!r})") + f"+ {w.cv_career!r} * {career}) / {total!r})") def _by_cv(expression: str) -> str: @@ -1888,27 +1892,34 @@ class Store: def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int, experience: int, career: int, strengths: Sequence[str], - gaps: Sequence[str]) -> bool: + gaps: Sequence[str], musts: Sequence[Mapping[str, str]] = ()) -> bool: """Store `judge`'s reading of one role for one agent, replacing the - one before, with `cv_score` (`app.models.cv_score`, at the default weights: the - table's column is not read, see `judgement`) and its tries - back at 0. False for an unknown role or agent.""" + one before: the three numbers as the model gave them, the verdict on + each of the agent's must-haves (`musts`, `{id, text, verdict}`) and how + many are broken, `cv_score` (`app.models.cv_score`, at the default + weights and with the cap: the table's column is not read, see + `judgement`) and its tries back at 0. False for an unknown role or + agent.""" + broken = sum(1 for m in musts if m.get("verdict") == "broken") try: with self.transaction() as conn: conn.execute( """INSERT INTO role_judgements (uid, agent_id, skills, experience, career, - cv_score, strengths, gaps, input_key, tries, - judged_at) - VALUES (?,?,?,?,?,?,?,?,?,0,?) + cv_score, strengths, gaps, musts, broken, + input_key, tries, judged_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,0,?) ON CONFLICT(uid, agent_id) DO UPDATE SET skills=excluded.skills, experience=excluded.experience, career=excluded.career, cv_score=excluded.cv_score, strengths=excluded.strengths, gaps=excluded.gaps, + musts=excluded.musts, broken=excluded.broken, input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""", (uid, agent_id, skills, experience, career, - cv_score(skills, experience, career), + cv_score(skills, experience, career, broken=broken), json.dumps(list(strengths), ensure_ascii=False), - json.dumps(list(gaps), ensure_ascii=False), input_key, self.now())) + json.dumps(list(gaps), ensure_ascii=False), + json.dumps([dict(m) for m in musts], ensure_ascii=False), broken, + input_key, self.now())) except sqlite3.IntegrityError: return False return True @@ -1935,7 +1946,8 @@ class Store: def judgement(self, agent_id: int | None, uid: str) -> Judgement | None: """The agent's judgement of one role, None when it has none, its `cv_score` the three numbers under the agent's weights as they are - now, not as stored.""" + now, not as stored, and `career_capped` the model's career as the + broken must-haves leave it (`BROKEN_CAP`).""" if agent_id is None: return None row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? " @@ -1944,8 +1956,9 @@ class Store: return None found = Judgement.from_row(row) agent = self.agent(agent_id) + found.career_capped = capped_career(found.career, found.broken) found.cv_score = cv_score(found.skills, found.experience, found.career, - agent.weights if agent else None) + agent.weights if agent else None, found.broken) return found def set_article_text(self, uid: str, text: str) -> Role | None: diff --git a/app/templates/settings/agents.html b/app/templates/settings/agents.html index cb25567..7771eb3 100644 --- a/app/templates/settings/agents.html +++ b/app/templates/settings/agents.html @@ -96,8 +96,8 @@ - Soft, read by the judge only: a role that breaks one of these scores low on career and the gap is named. Rules that must hold go under Where or Other rules instead. + + One must-have per line. The judge reports for each whether a posting meets it, clearly breaks it or does not say; a broken one caps the role's career score at 39 in code, whatever the model scored. Rules that must hold go under Where or Other rules instead.
How much each counts diff --git a/docs/decisions.md b/docs/decisions.md index c217ba2..7f6a14b 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -208,7 +208,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`extract_batch` and `summarise_batch` are `isolate=True`, with a timeout.** `context.check()` only runs between the calls `modeltask.one` makes, not inside one, so a call that hangs on either backend was otherwise unkillable short of restarting the server; a process boundary is what lets a Kill, or the timeout itself, actually end it. One call is already bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for summarise): a batch still running after that is stuck, not slow. The cost: an isolated job's context has no working `stop`, `task` or `fan_out` (see `pipeline/child.py`'s `context_for`), so a refusal or an account-level failure inside a batch can no longer stop the rest of that task's queued siblings, which run on to the same failure instead of being spared it. -**A third order, Best CV match, from a model reading each agent's best roles against its CV (2026-10-03).** Taken from the `/rank` of github.com/MadsLorentzen/ai-job-search, which has a model score a posting against a written profile on five dimensions. Fit compares topics; it cannot tell a step down, years of experience or a stated deal-breaker, which a model reading both texts can. An agent may hold a CV (pasted, or a `.md`/`.txt` file the browser reads into the box), its languages with a level (suggested from the CV by one model call, then edited), an "avoid / drains me" text and a "must-haves and deal-breakers" text; the prompts stay what the owner wants. The stage `judge` (after summarise in the pull, switched under Runs like extract, Haiku by default) reads the pairs of an enabled agent with a CV within its `judge_top` and `judge_min` limits, once per `input_key` (a hash of the CV, prompts, both texts and the role's text), and stores skills, experience and career (0 to 100, their rubric bands verbatim), 1 to 3 strengths and gaps in `role_judgements`, which rescoring and closing never touch. The CV match is the weighted mean of the three, with the weights per agent (`cv_skills`, `cv_experience`, `cv_career` in `agents.weights`, default 30, 25, 30) applied when a role is served, so changing them needs no model call. Culture is left out (guessed from ad text) and location too (the gates do it): a must-haves text that names a place makes the model mark location down, so places belong in the Where rules. Haiku does not always hold the deal-breaker band (0 to 39 for career); a stronger model or a clamp in code is the fix if that matters. Their scale is ten postings a run; ours is the top of a ranking that has already scored every role for free. +**A third order, Best CV match, from a model reading each agent's best roles against its CV (2026-10-03).** Taken from the `/rank` of github.com/MadsLorentzen/ai-job-search, which has a model score a posting against a written profile on five dimensions. Fit compares topics; it cannot tell a step down, years of experience or a stated deal-breaker, which a model reading both texts can. An agent may hold a CV (pasted, or a `.md`/`.txt` file the browser reads into the box), its languages with a level (suggested from the CV by one model call, then edited), an "avoid / drains me" text and a "must-haves and deal-breakers" text; the prompts stay what the owner wants. The stage `judge` (after summarise in the pull, switched under Runs like extract, Haiku by default) reads the pairs of an enabled agent with a CV within its `judge_top` and `judge_min` limits, once per `input_key` (a hash of the CV, prompts, both texts and the role's text), and stores skills, experience and career (0 to 100, their rubric bands verbatim), 1 to 3 strengths and gaps in `role_judgements`, which rescoring and closing never touch. The CV match is the weighted mean of the three, with the weights per agent (`cv_skills`, `cv_experience`, `cv_career` in `agents.weights`, default 30, 25, 30) applied when a role is served, so changing them needs no model call. Culture is left out (guessed from ad text) and location too (the gates do it): a must-haves text that names a place makes the model mark location down, so places belong in the Where rules. The model reports facts and code applies the rules: the must-haves text is read as one must-have per line (leading bullets and numbers dropped), the judge's profile numbers them M1..Mn, and for each judgement the model answers `musts`, a verdict per must-have, "met", "broken" only when the posting text clearly contradicts it, or "unclear" when the text does not say (never inferred from silence; a must-have the model leaves out counts as unclear). It scores career on its rubric as before and the career is stored as given. The cap is code's: when a role is served, any "broken" verdict makes career count as `min(career, BROKEN_CAP)` with `BROKEN_CAP = 39` (`app.models`), in `cv_score`, in `Store.judgement` (which serves the model's career, `career_capped` and the must-haves with their text) and in `store._cv_sql`, which reads a `broken` count stored beside the verdicts (`role_judgements.musts` and `.broken`, migration 29), so Best CV match orders by the capped value too. The verdicts store each line's text, so the reader still shows the must-haves a judgement was made against after the text is edited. The judge's `input_key` carries a format version (`judge.FORMAT`, 2), so judgements made before the verdicts existed are made again; until then they serve with no must-haves and no cap. The reader shows each must-have with ✓ met, ✗ broken or ? unclear, and career as "39 (capped: breaks a must-have; model said 55)". Their scale is ten postings a run; ours is the top of a ranking that has already scored every role for free. **A posting's languages and deadline, and a flag rather than a score.** Extract reads `languages_required` as objects (language, level on the ladder basic, conversational, professional, fluent, native, and required or plus) and a deadline, from the opening and the closing lines of the text, since both usually sit at the end. Jobnet and Jobbank state a deadline, kept as `deadline_host`; prep composes `deadline` host first. The language flag is computed when a role is served from the posting's languages and the agent's: ⚠ when a required one is missing or its level is above the agent's, ★ when a plus one is the agent's; it never changes a score. Closing soon is the deadline from today to today plus four days, in UTC. diff --git a/tests/test_companies.py b/tests/test_companies.py index 0e4e4ee..8ce835d 100644 --- a/tests/test_companies.py +++ b/tests/test_companies.py @@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) -> "VALUES (?, ?, '', '')", (key, name)) conn.execute("PRAGMA user_version = 19") reopened = Store(path) - assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28] + assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29] assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])] assert len(reopened.companies()) == 3 @@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp conn.execute("ALTER TABLE companies DROP COLUMN all_countries") conn.execute("PRAGMA user_version = 20") reopened = Store(path) - assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28] + assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29] # The tag of 21 is All countries and All roles since 24. tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0} assert {c.key: c.all_roles for c in reopened.companies()} == tagged diff --git a/tests/test_cv_feed.py b/tests/test_cv_feed.py index 2965290..ac47c6a 100644 --- a/tests/test_cv_feed.py +++ b/tests/test_cv_feed.py @@ -91,6 +91,37 @@ def test_best_cv_match_puts_the_judged_first_by_cv_score( "order": "best"}).status_code == 422 +def test_a_broken_must_have_caps_career_in_the_order_and_the_reader( + client: TestClient, store: Store) -> None: + owner, uid = setup(store) + # c: 90 on all three, but one must-have broken: career counts as 39. + musts = [{"id": "M1", "text": "Permanent", "verdict": "met"}, + {"id": "M2", "text": "Remote", "verdict": "broken"}] + store.set_judgement(owner.id, uid["c"], "k", 90, 90, 90, [], [], musts) + capped = round((30 * 90 + 25 * 90 + 30 * 39) / 85, 1) + rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"] + assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["c"], capped), (uid["b"], 60.0)] + # With career weighted up the capped value drops under b's 60. + store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=owner.cv_text, + weights=Weights(cv_skills=0, cv_experience=0, cv_career=1)) + rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"] + assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["b"], 60.0), (uid["c"], 39.0)] + one = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json() + j = one["judgement"] + assert (j["career"], j["career_capped"], j["broken"]) == (90, 39, 1) + assert j["musts"] == musts and one["cv_score"] == j["cv_score"] == 39.0 + + +def test_a_met_or_unclear_must_have_leaves_career_as_the_model_gave_it( + client: TestClient, store: Store) -> None: + owner, uid = setup(store) + musts = [{"id": "M1", "text": "Permanent", "verdict": "met"}, + {"id": "M2", "text": "Remote", "verdict": "unclear"}] + store.set_judgement(owner.id, uid["c"], "k", 90, 90, 90, [], [], musts) + j = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()["judgement"] + assert (j["career"], j["career_capped"], j["broken"], j["cv_score"]) == (90, 90, 0, 90.0) + + def test_the_agents_weights_make_the_cv_score_at_read_time( client: TestClient, store: Store) -> None: owner, uid = setup(store) diff --git a/tests/test_cv_fields.py b/tests/test_cv_fields.py index ae347e3..42311e2 100644 --- a/tests/test_cv_fields.py +++ b/tests/test_cv_fields.py @@ -16,7 +16,7 @@ from pydantic import ValidationError from app import prep from app.config import Config from app.jobs import extract -from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag +from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag, must_lines from app.storage import Store from desk import modeltask from desk.models import Item @@ -93,7 +93,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path: migrated = Store(path) - assert migrated.migrations_applied == [28] + assert migrated.migrations_applied == [28, 29] rows = {r["source_id"]: r for r in migrated._rows( "SELECT source_id, languages_required, deadline_host, deadline FROM roles")} assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None) @@ -105,7 +105,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path: assert migrated.role(posting("a").uid).deadline == "2026-10-18" with sqlite3.connect(path) as conn: conn.execute("PRAGMA user_version = 27") - assert Store(path).migrations_applied == [28] + assert Store(path).migrations_applied == [28, 29] # ---------------------------------------------------------------- deadline @@ -236,3 +236,25 @@ def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None: (json.dumps([{"language": "da", "level": "fluent", "kind": "required"}, {"language": "en", "level": "superb"}]), item.uid)) assert [lang.language for lang in store.role(item.uid).languages_required] == ["da"] + + +def test_v29_adds_the_musts_and_broken_columns_and_runs_twice(tmp_path: Path) -> None: + path = tmp_path / "v28.db" + Store(path) + with sqlite3.connect(path) as conn: + for column in ("musts", "broken"): + conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}") + conn.execute("PRAGMA user_version = 28") + + assert Store(path).migrations_applied == [29] + with sqlite3.connect(path) as conn: + columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")} + assert {"musts", "broken"} <= columns + conn.execute("PRAGMA user_version = 28") + assert Store(path).migrations_applied == [29] + + +def test_the_musts_text_is_one_must_have_per_line() -> None: + assert must_lines("- A\n* B c\n\u2022 D\n1. E\n2) F\n \nG\n-H") == [ + "A", "B c", "D", "E", "F", "G", "-H"] + assert must_lines("") == [] diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py index 0ccee19..ab7b5eb 100644 --- a/tests/test_dupes_storage.py +++ b/tests/test_dupes_storage.py @@ -261,7 +261,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None: migrated = Store(path) - assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28] + assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] assert dup_of(migrated) == {first.uid: None, second.uid: first.uid} with sqlite3.connect(path) as conn: assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'" diff --git a/tests/test_fetch_terms.py b/tests/test_fetch_terms.py index 408a59d..f4a700c 100644 --- a/tests/test_fetch_terms.py +++ b/tests/test_fetch_terms.py @@ -434,7 +434,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None: conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms") conn.execute("PRAGMA user_version = 22") migrated = Store(path) - assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28] + assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29] assert migrated.agents()[0].fetch_terms == [] diff --git a/tests/test_judge.py b/tests/test_judge.py index 3ff2d20..fdf9441 100644 --- a/tests/test_judge.py +++ b/tests/test_judge.py @@ -4,6 +4,8 @@ the stage. Nothing here reaches a model.""" from __future__ import annotations +import hashlib +import json from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -109,6 +111,64 @@ def test_a_run_writes_the_judgement_and_its_total( assert "- AI lead: Head of AI" in system and "deal-breaker" in system +def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + owner = hunter(store) + store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV, + must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n") + [uid] = ranked(store, owner, "1") + + def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [{**one, "career": 80, "musts": [ + {"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"}, + {"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]} + for one in judged(items)] + + model = stub(monkeypatch, reply) + modeltask.run(judge.TASK, config, store) + + [system] = model.systems + assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system + found = store.judgement(owner.id, uid) + assert found is not None + assert [(m.id, m.text, m.verdict) for m in found.musts] == [ + ("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"), + ("M3", "Danish team", "unclear")] + assert (found.career, found.career_capped, found.broken) == (80, 39, 1) + assert found.cv_score == round((30 * 80 + 25 * 70 + 30 * 39) / 85, 1) + + +def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + owner = hunter(store) + [uid] = ranked(store, owner, "1") + stub(monkeypatch) # answers without any `musts` + modeltask.run(judge.TASK, config, store) + found = store.judgement(owner.id, uid) + assert found is not None + assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")] + assert (found.career, found.career_capped, found.broken) == (90, 90, 0) + + +def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None: + item = judge.SCHEMA["properties"]["judgements"]["items"] + assert "musts" in item["required"] + verdict = item["properties"]["musts"]["items"]["properties"]["verdict"] + assert verdict["enum"] == ["met", "broken", "unclear"] + assert "career 0-39" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM + + +def test_judgements_made_before_the_format_are_made_again(store: Store) -> None: + owner = hunter(store) + assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text") + + +def hashlib_old_key(agent: Any, text: str) -> str: + blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts], + agent.avoid_text, agent.must_text, text], ensure_ascii=False) + return hashlib.sha1(blob.encode()).hexdigest()[:16] + + def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) -> None: for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})): ranked(store, hunter(store, name, **kwargs), name) diff --git a/tests/test_overrides.py b/tests/test_overrides.py index a6a5cd2..75cd0c9 100644 --- a/tests/test_overrides.py +++ b/tests/test_overrides.py @@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'") conn.execute("PRAGMA user_version = 23") migrated = Store(path) - assert migrated.migrations_applied == [24, 25, 26, 27, 28] + assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29] flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()} assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)} params = {s.target: s.params for s in migrated.subscriptions(source="ats")} diff --git a/tests/test_prep.py b/tests/test_prep.py index 4eb4423..843b65e 100644 --- a/tests/test_prep.py +++ b/tests/test_prep.py @@ -316,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None: migrated = Store(path) - assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28] + assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29] one = row(migrated, "1") assert (one["seniority_host"], one["employment_host"], one["remote_host"], one["country_host"]) == ("senior", "full_time", "true", "DK") diff --git a/tests/test_remote.py b/tests/test_remote.py index 93eb422..905807a 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -530,7 +530,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path: migrated = Store(path) - assert migrated.migrations_applied == [27, 28] + assert migrated.migrations_applied == [27, 28, 29] assert migrated.gazetteer().anywhere is not None row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles " "WHERE uid = ?", (board,)) @@ -540,7 +540,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path: assert area(migrated, search.uid) == ("", "") with sqlite3.connect(path) as conn: conn.execute("PRAGMA user_version = 26") - assert Store(path).migrations_applied == [27, 28] + assert Store(path).migrations_applied == [27, 28, 29] def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None: diff --git a/tests/test_storage.py b/tests/test_storage.py index a4c30e1..962c079 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 28 + assert LATEST == 29 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -168,6 +168,8 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None: conn.execute(f"ALTER TABLE companies DROP COLUMN {column}") for table in ("vectors", "role_tags", "agent_companies", "role_votes"): conn.execute(f"DROP TABLE {table}") + for column in ("musts", "broken"): # v29 + conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}") conn.execute("PRAGMA user_version = 1") assert Store(old).migrations_applied == list(range(2, LATEST + 1)) @@ -187,7 +189,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated( migrated = Store(path) - assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28] + assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] acme = migrated.company("acme") assert acme is not None assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example") @@ -392,7 +394,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28] + assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -416,7 +418,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None migrated = Store(path) - assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28] + assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] acme = migrated.company("acme") assert acme is not None assert (acme.careers_url, acme.notes, acme.hand_reason) == ( @@ -445,7 +447,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( migrated = Store(path) - assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28] + assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] with sqlite3.connect(path) as conn: rows = {uid: (fold, level) for uid, fold, level in conn.execute( "SELECT uid, place_fold, seniority FROM roles")} @@ -688,7 +690,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> migrated = Store(path) - assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28] + assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"] assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"] assert migrated.role_places(stated.uid) == [] @@ -730,7 +732,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level( migrated = Store(path) - assert migrated.migrations_applied == [25, 26, 27, 28] + assert migrated.migrations_applied == [25, 26, 27, 28, 29] with sqlite3.connect(path) as conn: found = {sid: rest for sid, *rest in conn.execute( "SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority " @@ -748,12 +750,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path) migrated = Store(path) - assert migrated.migrations_applied == [26, 27, 28] + assert migrated.migrations_applied == [26, 27, 28, 29] role = migrated.role(posting("a").uid) assert role is not None and role.posting_url == "" with sqlite3.connect(path) as conn: conn.execute("PRAGMA user_version = 25") # the column is there already - assert Store(path).migrations_applied == [26, 27, 28] + assert Store(path).migrations_applied == [26, 27, 28, 29] def test_a_pull_keeps_the_posting_url(store: Store) -> None: