Give an agent a CV, languages, avoid and must texts, and read a posting's languages with a level and its deadline (migration 28)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-10-03 21:54:16 +02:00
1 parent ed6d641373
commit 4fc474c437
15 files changed
+716 -79

No files matched your search

+76 -18
View File
@@ -10,8 +10,9 @@ refused batch, and failing with a notice when no model is reachable are
`extract` is also the afterburner for the level and the remote area. A role
is waiting while it is open, has not been given up on, has never been
answered (`extracted_at IS NULL`), has no model level (`seniority_model`
empty) or was never asked where a remote hire may live (`remote_where_model`
NULL), and is in the best of an enabled agent's
empty), was never asked where a remote hire may live (`remote_where_model`
NULL) or for its languages and deadline (`languages_required` NULL), and is
in the best of an enabled agent's
ungated scores as deep and as low as that agent's own `limits` say
(`extract_top`, `extract_min`). That bound is what keeps the cost
proportional to the rankings rather than to the boards:
@@ -47,7 +48,10 @@ from pipeline import NULL, RunContext, Task, job
from app import prep
from app.config import Config
from app.jobs import stages
from app.models import LEVELS, PERMITS, REMOTE, SENIORITIES, ExtractResult, Role
from app.models import (
LANGUAGE_LEVELS, LEVELS, PERMITS, REMOTE, SENIORITIES, ExtractResult, Role, iso_day,
language_code,
)
from app.storage import Store
# `FUNCTIONS` is the `ats` plugin's vocabulary, reached the way `web/pages.py`
@@ -58,10 +62,13 @@ if str(_ROOT) not in sys.path:
from plugins.ats import FUNCTIONS # noqa: E402
#: How much of the description goes in one item. Salary, place and remote are
#: nearly always in the opening or the closing lines of an ad; the first 3000
#: characters hold most of them at a fifth of a long posting's tokens.
EXCERPT = 3000
#: How much of the description goes in one item: the opening `HEAD` and the
#: closing `TAIL` characters, joined by an ellipsis when the text is longer.
#: Salary, place and remote are nearly always in the opening lines of an ad,
#: and the languages and the deadline in the closing ones.
HEAD = 2500
TAIL = 1500
EXCERPT = HEAD + TAIL
SYSTEM = f"""You read job postings and fill in structured fields, one answer per posting.
@@ -116,11 +123,32 @@ Fields:
- function: one of {", ".join(FUNCTIONS)}, or ''.
- permit: "sponsors" if the posting says it sponsors a visa or work permit,
"no_sponsorship" if it says it does not, else ''.
- languages_required: ISO 639-1 codes ("de", "en") of the languages the posting
says are required. Not ones it calls a plus. [] when it names none."""
- languages_required: every language the posting asks the hire to speak, one
entry each: `language` its ISO 639-1 code ("da", "en", "de"), `level` the
level it asks for, one of {", ".join(LANGUAGE_LEVELS)} (lowest first), or ''
when it states none, and `kind` "required" when the posting requires it,
"plus" when it calls it an advantage or nice to have. "Mother tongue" is
native; "fluent", "excellent" or "business fluent" is fluent; "very good",
"strong" or "professional working" is professional; "good", "working
knowledge" or "conversational" is conversational. The language the posting
is written in is not a requirement unless it says so. [] when it names none.
- deadline: the last day to apply as YYYY-MM-DD when the posting states a
date ("Ansøgningsfrist: 18.10.2026", "apply by 15 October"); with no year,
the first such date after `posted`. '' for none, "as soon as possible" or a
rolling review ("løbende")."""
_TEXT = {"type": "string"}
_SALARY = {"type": ["integer", "null"]}
_LANGUAGE_ITEM: dict[str, Any] = {
"type": "object",
"properties": {
"language": _TEXT,
"level": {"type": "string", "enum": ["", *LANGUAGE_LEVELS]},
"kind": {"type": "string", "enum": ["required", "plus"]},
},
"required": ["language", "level", "kind"],
"additionalProperties": False,
}
SCHEMA: dict[str, Any] = {
"type": "object",
@@ -141,11 +169,12 @@ SCHEMA: dict[str, Any] = {
"currency": _TEXT,
"function": {"type": "string", "enum": ["", *FUNCTIONS]},
"permit": {"type": "string", "enum": list(PERMITS)},
"languages_required": {"type": "array", "items": _TEXT},
"languages_required": {"type": "array", "items": _LANGUAGE_ITEM},
"deadline": _TEXT,
},
"required": ["uid", "place", "country", "remote", "remote_where", "seniority",
"salary_min", "salary_max", "currency", "function",
"permit", "languages_required"],
"permit", "languages_required", "deadline"],
"additionalProperties": False,
},
},
@@ -156,7 +185,6 @@ SCHEMA: dict[str, Any] = {
_COUNTRY = re.compile(r"^[A-Z]{2}$")
_CURRENCY = re.compile(r"^[A-Z]{3}$")
_LANGUAGE = re.compile(r"^[a-z]{2}$")
# ------------------------------------------------------------------- batches
@@ -164,12 +192,14 @@ _LANGUAGE = re.compile(r"^[a-z]{2}$")
def _item(role: Role) -> dict[str, Any]:
"""One posting as the model sees it. `location` is the board's own place
text where it gave one: part of the posting, and often the only line that
names the country."""
text = " ".join(role.description.split())[:EXCERPT]
names the country. `posted` is the day the role counts as posted, for a
deadline that names no year."""
text = excerpt(role.description)
return {key: value for key, value in {
"uid": role.uid,
"title": role.title,
"location": role.place,
"posted": role.posted_at[:10],
"stated": role.seniority_stated,
"title_level": role.seniority_title,
"text": text,
@@ -177,6 +207,15 @@ def _item(role: Role) -> dict[str, Any]:
}.items() if value}
def excerpt(text: str) -> str:
"""The text with its whitespace collapsed: whole up to `EXCERPT`
characters, else its opening `HEAD` and closing `TAIL` characters."""
flat = " ".join(text.split())
if len(flat) <= EXCERPT:
return flat
return f"{flat[:HEAD]} … {flat[-TAIL:]}"
def build_batch(config: Config, store: Store, limit: int = 20,
uids: Sequence[str] | None = None) -> dict[str, Any]:
return {"instructions": SYSTEM,
@@ -213,12 +252,29 @@ def _names(value: Any) -> list[str]:
return list(dict.fromkeys(name for name in found if name))[:_WHERE_NAMES]
def _languages(value: Any) -> list[dict[str, str]]:
"""`languages_required` checked: an entry per code, the first kept, with
a level of the ladder or '', and `required` unless it says `plus`. A
bare code, as an older reply gave, is a requirement with no level."""
found: dict[str, dict[str, str]] = {}
for raw in value if isinstance(value, list) else []:
one = raw if isinstance(raw, dict) else {"language": raw}
code = language_code(one.get("language"))
if not code or code in found:
continue
found[code] = {"language": code,
"level": _pick(one.get("level"), LANGUAGE_LEVELS),
"kind": "plus" if one.get("kind") == "plus" else "required"}
return list(found.values())
def fields(answer: dict[str, Any]) -> dict[str, Any]:
"""One answer, checked field by field. Anything outside a field's
vocabulary is '' or None, which `set_extracted` never writes, except
for `remote` and `remote_where`, which it writes as they come."""
languages = [code for raw in answer.get("languages_required") or []
if (code := _code(raw, _LANGUAGE, upper=False))]
for `remote`, `remote_where`, `languages_required` and `deadline`, which
it writes as they come. `languages` is the required codes, for the
`languages` gate."""
named = _languages(answer.get("languages_required"))
return {
"place": " ".join(str(answer.get("place") or "").split())[:200],
"country": _code(answer.get("country"), _COUNTRY, upper=True),
@@ -230,7 +286,9 @@ def fields(answer: dict[str, Any]) -> dict[str, Any]:
"currency": _code(answer.get("currency"), _CURRENCY, upper=True),
"function": _pick(answer.get("function"), FUNCTIONS),
"permit": _pick(answer.get("permit"), PERMITS),
"languages": list(dict.fromkeys(languages)),
"languages": [one["language"] for one in named if one["kind"] == "required"],
"languages_required": named,
"deadline": iso_day(answer.get("deadline")),
}
+202 -2
View File
@@ -14,6 +14,7 @@ import json
import re
import sys
import unicodedata
from datetime import date
from functools import lru_cache
from pathlib import Path
from typing import Annotated, Any, Literal, get_args
@@ -782,6 +783,36 @@ AFFINITY = range(-1, 3)
HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"]
HAND_REASONS: tuple[str, ...] = get_args(HandReason)
#: How well somebody speaks a language, lowest first: what an agent's CV
#: states and what a posting asks for. The order is the comparison.
LanguageLevel = Literal["basic", "conversational", "professional", "fluent", "native"]
LANGUAGE_LEVELS: tuple[str, ...] = get_args(LanguageLevel)
#: Whether a posting requires a language or calls it a plus.
LanguageKind = Literal["required", "plus"]
#: English names of the languages a posting in reach names, by ISO 639-1
#: code, for `language_flag`'s note. A code not here is shown as it is.
LANGUAGE_NAMES: dict[str, str] = {
"ar": "Arabic", "bg": "Bulgarian", "cs": "Czech", "da": "Danish", "de": "German",
"el": "Greek", "en": "English", "es": "Spanish", "et": "Estonian", "fi": "Finnish",
"fo": "Faroese", "fr": "French", "hi": "Hindi", "hr": "Croatian", "hu": "Hungarian",
"is": "Icelandic", "it": "Italian", "ja": "Japanese", "kl": "Greenlandic",
"ko": "Korean", "lt": "Lithuanian", "lv": "Latvian", "nb": "Norwegian",
"nl": "Dutch", "nn": "Norwegian", "no": "Norwegian", "pl": "Polish",
"pt": "Portuguese", "ro": "Romanian", "ru": "Russian", "sk": "Slovak",
"sl": "Slovenian", "sr": "Serbian", "sv": "Swedish", "tr": "Turkish",
"uk": "Ukrainian", "zh": "Chinese",
}
_LANGUAGE_CODE = re.compile(r"^[a-z]{2}$")
def language_code(value: Any) -> str:
"""An ISO 639-1 code, lowercased, or '' when `value` is not one."""
code = str(value or "").strip().lower()
return code if _LANGUAGE_CODE.match(code) else ""
# -------------------------------------------------------------- posting dates
@@ -847,6 +878,16 @@ def posted_at(published_at: str, first_seen: str, host: str) -> str:
return first_seen
def iso_day(value: Any) -> str:
"""A date as `YYYY-MM-DD` from an ISO date or timestamp ("2026-10-18",
"2026-10-18T00:00:00"), or '' when `value` holds no real date."""
text = str(value or "").strip()[:10]
try:
return date.fromisoformat(text).isoformat() if len(text) == 10 else ""
except ValueError:
return ""
def _json(value: Any, empty: str) -> Any:
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
value that is already parsed passes, and one that does not parse is the
@@ -863,6 +904,129 @@ def _json(value: Any, empty: str) -> Any:
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
def _json_list(value: Any) -> list[Any]:
"""A nullable JSON array column as a list: NULL, a parse failure and
anything but a list are []."""
found = _json(value, "[]") if value is not None else []
return found if isinstance(found, list) else []
class CvLanguage(Model):
"""A language an agent's CV states, with how well: `language` an ISO
639-1 code, `level` one of `LANGUAGE_LEVELS`."""
language: str
level: LanguageLevel
@field_validator("language", mode="before")
@classmethod
def _code(cls, value: Any) -> str:
code = language_code(value)
if not code:
raise ValueError(f"a language is an ISO 639-1 code such as 'en', not {value!r}")
return code
def cv_languages(value: Any) -> list[CvLanguage]:
"""An agent's languages as stored and sent: one per code, the first
kept, in the order given."""
seen: set[str] = set()
out: list[CvLanguage] = []
for item in _json_list(value):
found = item if isinstance(item, CvLanguage) else CvLanguage.model_validate(item)
if found.language not in seen:
seen.add(found.language)
out.append(found)
return out
#: An agent's languages, normalised by `cv_languages`.
CvLanguages = Annotated[list[CvLanguage], BeforeValidator(cv_languages)]
class RoleLanguage(Model):
"""A language a posting names (`roles.languages_required`, read by
`extract`): its ISO 639-1 code, the level asked for ('' when the posting
does not say), and whether it is required or a plus."""
language: str
level: Literal["", "basic", "conversational", "professional", "fluent", "native"] = ""
kind: LanguageKind = "required"
def _role_languages(value: Any) -> list[RoleLanguage]:
out = []
for item in _json_list(value):
try:
out.append(item if isinstance(item, RoleLanguage)
else RoleLanguage.model_validate(item))
except ValueError:
continue
return out
#: `roles.languages_required`: NULL (never read) and [] are both no language.
RoleLanguages = Annotated[list[RoleLanguage], BeforeValidator(_role_languages)]
LangFlag = Literal["missing", "below", "plus", ""]
def _language_name(code: str) -> str:
return LANGUAGE_NAMES.get(code, code)
def _asked(language: RoleLanguage) -> str:
name = _language_name(language.language)
return f"{name} ({language.level})" if language.level else name
def language_flag(required: list[RoleLanguage],
mine: list[CvLanguage]) -> tuple[LangFlag, str]:
"""`(flag, note)` for a posting's languages against an agent's.
`missing` when a required language is not one of the agent's, `below`
when the level it states is above the agent's, else `plus` when a plus
language is one of the agent's, else ''. The note says why, as
"requires Danish (fluent); yours: none". No flag when the agent states
no languages: there is nothing to compare with."""
if not mine:
return "", ""
levels = {m.language: m.level for m in mine}
rank = LANGUAGE_LEVELS.index
needed = [r for r in required if r.kind == "required"]
missing = [r for r in needed if r.language not in levels]
if missing:
return "missing", "; ".join(f"requires {_asked(r)}; yours: none" for r in missing)
below = [r for r in needed if r.level and rank(r.level) > rank(levels[r.language])]
if below:
return "below", "; ".join(f"requires {_asked(r)}; yours: {levels[r.language]}"
for r in below)
plus = [r for r in required if r.kind == "plus" and r.language in levels]
if plus:
return "plus", "; ".join(f"{_asked(r)} is a plus; yours: {levels[r.language]}"
for r in plus)
return "", ""
def cv_score(skills: int, experience: int, career: int) -> float:
"""The one number of a CV match, 0 to 100: skills and career weigh 30,
experience 25 (docs/decisions.md, the CV match)."""
return round((30 * skills + 25 * experience + 30 * career) / 85, 1)
class Judgement(Model):
"""What `judge` read of one role for one agent (`role_judgements`): the
three dimensions 0 to 100, `cv_score` (`cv_score`), and 1 to 3 strengths
and gaps grounded in the posting."""
skills: int
experience: int
career: int
cv_score: float
strengths: JsonStrings = []
gaps: JsonStrings = []
judged_at: str
# ------------------------------------------------------------------ models
class RoleCopy(Model):
@@ -924,8 +1088,15 @@ class Role(Model):
employment: str = ""
function: str = ""
permit: Permit = ""
#: ISO 639-1 codes the posting says are required, lowercase.
#: ISO 639-1 codes the posting says are required, lowercase: what the
#: `languages` gate reads.
languages: JsonStrings = []
#: Every language the posting names, with the level it asks for and
#: whether it is required or a plus (`extract`); [] until read.
languages_required: RoleLanguages = []
#: The last day to apply, `YYYY-MM-DD`: the host's own date, else the
#: one `extract` read; None when neither says (`app.prep`).
deadline: str | None = None
description: str = ""
feed_text: str = ""
#: The short text the host's list carries, until `description` arrives.
@@ -1013,6 +1184,13 @@ class ScoredRole(Role):
#: The agent's prompt that gave the role its fit; "" when fit did not
#: count (gated, no vector, no prompt with text).
prompt: str = ""
#: The agent's `judge` reading of the role against its CV, None until
#: judged; `cv_score` is its total.
judgement: Judgement | None = None
cv_score: float | None = None
#: The posting's languages against the agent's (`language_flag`).
lang_flag: LangFlag = ""
lang_note: str = ""
#: How many characters `RoleRow.excerpt` keeps at most: the feed row's own
@@ -1087,6 +1265,13 @@ class RoleRow(Model):
#: As `Role.remote_where`: the area a remote hire may live in, '' when
#: the role is not remote or does not say.
remote_where: str = ""
#: As `Role.deadline`: the last day to apply, None when nothing says.
deadline: str | None = None
#: As `ScoredRole`: the posting's languages against the agent's, and
#: the total of the agent's `judge` reading, None until judged.
lang_flag: LangFlag = ""
lang_note: str = ""
cv_score: float | None = None
@computed_field # type: ignore[prop-decorator]
@property
@@ -1238,6 +1423,10 @@ class Limits(Model):
extract_min: float = Field(0.55, ge=0, le=1)
summarise_top: int = Field(30, ge=0)
summarise_min: float = Field(0.65, ge=0, le=1)
#: How deep `judge` reads an agent with a CV: a long answer per role,
#: so between summarise's 30 and extract's 500.
judge_top: int = Field(150, ge=0)
judge_min: float = Field(0.55, ge=0, le=1)
class Prompt(Model):
@@ -1299,7 +1488,14 @@ class Agent(Model):
`fetch_terms` are words a role must match for a big board read with the
wide reach to send it (`app/jobs/reach.py`): the pull's terms are the
union over the enabled agents, and an agent with none means no board is
narrowed by words. They decide what is fetched, not what ranks."""
narrowed by words. They decide what is fetched, not what ranks.
`cv_text`, `cv_languages`, `avoid_text` and `must_text` are what
`judge` reads a role against: the CV, the languages it states with a
level, what drains the person, and their must-haves and deal-breakers
(soft, read by the model only; a hard rule is a gate). An agent without
a CV is not judged, and one without languages flags none
(`language_flag`)."""
id: int
name: str
@@ -1308,6 +1504,10 @@ class Agent(Model):
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
limits: Annotated[Limits, BeforeValidator(lambda v: _json(v, "{}"))] = Limits()
fetch_terms: FetchTerms = []
cv_text: str = ""
cv_languages: CvLanguages = []
avoid_text: str = ""
must_text: str = ""
enabled: bool = True
created_at: str
+7 -3
View File
@@ -5,7 +5,8 @@ and marks the role due. `run` maps every due role: its company through
`company_aliases`, its places against the gazetteer, its level, contract
and remote through the host's tables (`app.prep.words`), its country from
the places (`app.prep.country`), and a remote role's area
(`app.prep.area`). `remote` is the model's (`remote_model`), else the
(`app.prep.area`), and its `deadline` (the host's date, else the one
`extract` read). `remote` is the model's (`remote_model`), else the
host's word, else what the place line or a search role's title says. When
the tables or the gazetteer in code (`places.seed`) changed since the last
run (`fingerprint`, kept in state as `prep:tables`), the new places and
@@ -35,8 +36,9 @@ from app.storage import Store
#: Raised when the rules in this package change the way a stored word maps,
#: so every open role is mapped again as it is after a table changes. 3: the
#: remote area, and remote as the model's before the host's word.
VERSION = 3
#: remote area, and remote as the model's before the host's word. 4: the
#: deadline, the host's before the model's.
VERSION = 4
#: The state key the last run's `fingerprint` is kept under.
STATE = "prep:tables"
@@ -115,6 +117,7 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
if remote == "remote" else ()),
"country": country(gaz, place_ids, row["country_host"]),
"deadline": row["deadline_host"] or row["deadline_model"] or None,
}
@@ -124,6 +127,7 @@ def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
or found["remote"] != row["remote"] \
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
or (found["country"] and found["country"] != row["country"]) \
or found["deadline"] != row["deadline"] \
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
"seniority_title", "employment"))
+100 -7
View File
@@ -19,7 +19,8 @@ from collections.abc import Callable
from app import places
from app.models import (
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
SENIORITIES, company_fold, company_key, country_of, host_word, iso_day, posted_at,
posted_host,
)
from app.seniority import from_title
from app.storage import gazetteer
@@ -120,7 +121,11 @@ CREATE TABLE IF NOT EXISTS roles (
employment TEXT NOT NULL DEFAULT '',
prep_due INTEGER NOT NULL DEFAULT 0,
remote_model TEXT NOT NULL DEFAULT '',
remote_where_model TEXT
remote_where_model TEXT,
languages_required TEXT,
deadline_host TEXT NOT NULL DEFAULT '',
deadline_model TEXT NOT NULL DEFAULT '',
deadline TEXT
);
-- The *_host columns hold the host's own words as a reader handed them on
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
@@ -134,9 +139,14 @@ CREATE TABLE IF NOT EXISTS roles (
-- word mapped to app.seniority's vocabulary, and the title's. remote_model is
-- extract's remote, which wins over the host's word and the place line, and
-- remote_where_model the names it read for where a remote hire may live (a
-- JSON list, NULL never asked, [] not stated). No comment sits among the last
-- columns: SQLite cannot drop a column a comment precedes, which the
-- migration tests do.
-- JSON list, NULL never asked, [] not stated). languages_required is every
-- language extract read in the posting, a JSON list of {language, level,
-- kind}, NULL never asked; languages keeps the required codes for the
-- languages gate. deadline_host is the host's own last day to apply, as a
-- pull hands it on, deadline_model the one extract read, both YYYY-MM-DD or
-- '', and deadline the first of them, NULL for none, written by prep. No
-- comment sits among the last columns: SQLite cannot drop a column a comment
-- precedes, which the migration tests do.
-- The gazetteer: a place is a continent, a country (key the ISO code), a
-- region or a city, in the place parent_id names. key is unique per kind:
@@ -317,7 +327,10 @@ END;
-- limits is how deep into this agent's ranking extract and summarise read,
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
-- means no narrowing.
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
-- level}), avoid_text (what drains) and must_text (must-haves and
-- deal-breakers) are what judge reads a role against; an agent with no CV
-- is not judged.
CREATE TABLE IF NOT EXISTS agents (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
@@ -326,7 +339,11 @@ CREATE TABLE IF NOT EXISTS agents (
enabled INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL,
limits TEXT NOT NULL DEFAULT '{}',
fetch_terms TEXT NOT NULL DEFAULT '[]'
fetch_terms TEXT NOT NULL DEFAULT '[]',
cv_text TEXT NOT NULL DEFAULT '',
cv_languages TEXT NOT NULL DEFAULT '[]',
avoid_text TEXT NOT NULL DEFAULT '',
must_text TEXT NOT NULL DEFAULT ''
);
-- What an agent looks for, as several named texts in order; a role's fit is
@@ -378,6 +395,29 @@ CREATE TABLE IF NOT EXISTS scores (
PRIMARY KEY (uid, agent_id)
);
-- One agent's judge reading of a role against its CV. Its own table, since
-- replace_scores rewrites scores every run and a judgement outlives that
-- and the role closing. skills, experience and career are 0 to 100,
-- cv_score their weighted total, strengths and gaps JSON lists. input_key
-- is a hash of the CV, prompts, avoid text and the role's text it read: a
-- different one is read again. tries counts replies that left the role
-- without one since; at 3 it is given up until the agent is saved again. A
-- row with judged_at NULL holds only tries.
CREATE TABLE IF NOT EXISTS role_judgements (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
skills INTEGER,
experience INTEGER,
career INTEGER,
cv_score REAL,
strengths TEXT NOT NULL DEFAULT '[]',
gaps TEXT NOT NULL DEFAULT '[]',
input_key TEXT NOT NULL DEFAULT '',
tries INTEGER NOT NULL DEFAULT 0,
judged_at TEXT,
PRIMARY KEY (uid, agent_id)
);
-- One vector per role, little-endian float32. Only comparable to another from
-- the same model, so every read is scoped by it.
CREATE TABLE IF NOT EXISTS vectors (
@@ -442,6 +482,10 @@ CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
CREATE INDEX IF NOT EXISTS idx_role_remote_place ON role_remote(place_id, uid);
-- The roles the next prep maps. Partial: after a prep there are none.
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
-- The feed's Closing soon filter. Partial: most roles state no deadline.
CREATE INDEX IF NOT EXISTS idx_roles_deadline ON roles(deadline) WHERE deadline IS NOT NULL;
-- An agent's judgements, for its feed under Best CV match.
CREATE INDEX IF NOT EXISTS idx_role_judgements_agent ON role_judgements(agent_id, cv_score DESC);
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -649,6 +693,11 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
# Anywhere. The areas are written by the next prep (app.prep.stage.VERSION 3).
(27, "roles.remote_model and roles.remote_where_model, role_remote and the Anywhere "
"place: where a remote hire may live", lambda conn: _add_remote_area(conn)),
# A Python step: it adds the columns only where they are missing. The
# deadline is composed by the next prep (app.prep.stage.VERSION 4).
(28, "agents.cv_text, cv_languages, avoid_text and must_text, roles.languages_required and the "
"deadline columns, and role_judgements: the CV match, the language flag and the "
"deadline", lambda conn: _add_cv_match(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -932,6 +981,50 @@ def _add_remote_area(conn: sqlite3.Connection) -> None:
gazetteer.seed(conn)
def _add_cv_match(conn: sqlite3.Connection) -> None:
"""Migration 28's step: an agent's CV, languages, avoid text and
must-haves, all empty; `roles.languages_required` NULL for every role, so `extract`
reads the roles it has read once more for the languages and the
deadline; `roles.deadline_host` from the date a reader already keeps in
`raw` (`applicationDeadline`, Jobnet and Jobbank), `deadline_model`
empty and `deadline` NULL until the next prep; and `role_judgements`,
empty. Runs twice without harm."""
agents = {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
for column, ddl in (("cv_text", "TEXT NOT NULL DEFAULT ''"),
("cv_languages", "TEXT NOT NULL DEFAULT '[]'"),
("avoid_text", "TEXT NOT NULL DEFAULT ''"),
("must_text", "TEXT NOT NULL DEFAULT ''")):
if column not in agents:
conn.execute(f"ALTER TABLE agents ADD COLUMN {column} {ddl}")
roles = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
for column, ddl in (("languages_required", "TEXT"),
("deadline_host", "TEXT NOT NULL DEFAULT ''"),
("deadline_model", "TEXT NOT NULL DEFAULT ''"),
("deadline", "TEXT")):
if column not in roles:
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} {ddl}")
dated = conn.execute(
"SELECT uid, json_extract(raw, '$.applicationDeadline') FROM roles "
"WHERE deadline_host = '' AND json_extract(raw, '$.applicationDeadline') IS NOT NULL"
).fetchall()
found = [(day, uid) for uid, value in dated if (day := iso_day(value))]
conn.executemany("UPDATE roles SET deadline_host = ? WHERE uid = ?", found)
log.info("migration 28: a host deadline on %d roles", len(found))
conn.execute("""CREATE TABLE IF NOT EXISTS role_judgements (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
skills INTEGER,
experience INTEGER,
career INTEGER,
cv_score REAL,
strengths TEXT NOT NULL DEFAULT '[]',
gaps TEXT NOT NULL DEFAULT '[]',
input_key TEXT NOT NULL DEFAULT '',
tries INTEGER NOT NULL DEFAULT 0,
judged_at TEXT,
PRIMARY KEY (uid, agent_id))""")
def _add_all_roles(conn: sqlite3.Connection) -> None:
"""Migration 21's step. The column; the tag for every company that is a
favourite of any agent, since favourite no longer widens what a pull
+66 -26
View File
@@ -24,10 +24,11 @@ from desk.models import Item
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
RoleCopy, RolePlace, RoleRow, ScoredRole, cv_languages as normal_languages,
language_flag,
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
fetch_terms as normal_terms, host_word, posted_at, posted_host, remote_of, title_key,
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
)
from app.storage import gazetteer, schema
@@ -46,7 +47,9 @@ _RAW_TEXT = ("place", "currency")
_HOST_WORDS = ("seniority", "employment", "remote", "country")
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
#: What prep maps from that a pull rewrites whatever it brings.
_PREP_OVERWRITTEN = ("title", "seniority_host")
#: `deadline_host` is the host's last day to apply (`applicationDeadline` in
#: `raw`, Jobnet and Jobbank), as `YYYY-MM-DD`.
_PREP_OVERWRITTEN = ("title", "seniority_host", "deadline_host")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
@@ -63,13 +66,16 @@ GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: still open, no copy of another
#: (`rebuild_dupes`), not given up, and either never answered, without a
#: model level, or never asked where a remote hire may live
#: (`remote_where_model` NULL). The last two make `extract` the afterburner
#: for the level and the area: a role read before they were asked for is
#: read again, and only fills what is still empty (`set_extracted`). A reply
#: model level, never asked where a remote hire may live
#: (`remote_where_model` NULL), or never asked for its languages and deadline
#: (`languages_required` NULL). The last three make `extract` the
#: afterburner for the level, the area, the languages and the deadline: a
#: role read before they were asked for is read again, and only fills what
#: is still empty (`set_extracted`). A reply
#: that names no level leaves the role waiting and is charged a try
#: (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL) "
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL "
"OR languages_required IS NULL) "
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
@@ -730,8 +736,10 @@ class Store:
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
`remote`, `country`) are left as prep last wrote them. `prep_due`
is 1 on a new role and grows by one when the title, the company, a
host word or the place texts changed or the role reopens, which
is what the next prep reads.
host word, the host's deadline or the place texts changed or the
role reopens, which is what the next prep reads. `deadline_host` is
`raw["applicationDeadline"]` as `YYYY-MM-DD` (`iso_day`), '' when
the pull states none, and prep composes `deadline` from it.
`posted_at` (see `app.models.posted_at`) is set from this pull's own
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
@@ -801,6 +809,7 @@ class Store:
# not kept-filled like _RAW_TEXT: a board that stops stating pay
# must be able to clear it, not leave a stale line behind.
row["salary_text"] = str(raw.get("salary_text") or "")
row["deadline_host"] = iso_day(raw.get("applicationDeadline"))
row["prep_due"] = 1
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
@@ -810,7 +819,7 @@ class Store:
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text", "seniority_host")]
"salary_text", "seniority_host", "deadline_host")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -1421,6 +1430,7 @@ class Store:
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country, remote_model, remote_where_model,
deadline_host, deadline_model, deadline,
json_extract(raw, '$.places') AS raw_places,
(SELECT json_group_array(place_id) FROM (
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
@@ -1446,8 +1456,8 @@ class Store:
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_area` and
`country`. `seniority` is the model's level, else the stated, else
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
the title's. `remote` is written as prep composed it (the model's,
else the host's word, else the place line) and `remote_area`, place
ids, replaces the role's `role_remote`. An empty `country` leaves the
@@ -1464,6 +1474,7 @@ class Store:
employment = :employment,
remote = :remote,
country = COALESCE(NULLIF(:country, ''), country),
deadline = :deadline,
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
WHERE uid = :uid""", mapped).rowcount
conn.executemany("DELETE FROM role_remote WHERE uid = ?",
@@ -1541,7 +1552,8 @@ class Store:
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
.replace("dup_of", "r.dup_of")
.replace("seniority_model", "r.seniority_model")
.replace("remote_where_model", "r.remote_where_model")]
.replace("remote_where_model", "r.remote_where_model")
.replace("languages_required", "r.languages_required")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
@@ -1595,7 +1607,10 @@ class Store:
`remote_model` and `remote_where_model` (a JSON list, `[]` for none),
even when empty, since an answer is what the model read this time;
`remote` and the area are composed from them by prep, so the role is
marked due (`prep.run_roles`, which `extract` calls)."""
marked due (`prep.run_roles`, which `extract` calls).
`languages_required` (a JSON list, `[]` for none) and the deadline,
as `deadline_model` (`YYYY-MM-DD` or ''), are written outright too;
prep composes `deadline` from the host's and this one."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1620,9 +1635,14 @@ class Store:
sets["languages"] = json.dumps(languages)
remote = str(fields.get("remote") or "")
where = [str(name) for name in fields.get("remote_where") or []]
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where)
named = list(fields.get("languages_required") or [])
deadline = str(fields.get("deadline") or "")
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where) \
+ bool(named) + bool(deadline)
sets["remote_model"] = remote
sets["remote_where_model"] = json.dumps(where)
sets["languages_required"] = json.dumps(named)
sets["deadline_model"] = deadline
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
@@ -1880,15 +1900,22 @@ class Store:
weights: Weights | None = None, enabled: bool = True,
agent_id: int | None = None, limits: Limits | None = None,
prompts: Sequence[Prompt] | None = None,
fetch_terms: Sequence[str] | None = None) -> Agent:
fetch_terms: Sequence[str] | None = None,
cv_text: str | None = None,
cv_languages: Sequence[CvLanguage] | None = None,
avoid_text: str | None = None,
must_text: str | None = None) -> Agent:
"""Create an agent, or with `agent_id` replace that one's fields and
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
named after the agent. Without `fetch_terms` an update keeps the
stored ones and a new agent has none, so a caller that does not know
them (the CLI's prompt edit) loses none. A prompt whose text is unchanged keeps its
cached vector, whatever its name or place, so an edit re-embeds only
the prompts it changed. Raises ValueError on an empty or taken name,
or two prompts of one name, KeyError on an unknown id."""
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
`avoid_text` and `must_text` are kept the same way. A prompt whose text is
unchanged keeps its cached vector, whatever its name or place, so an
edit re-embeds only the prompts it changed. A save gives every role
`judge` gave up on for the agent its tries back, since what it is
judged against may have changed. Raises ValueError on an empty or
taken name, or two prompts of one name, KeyError on an unknown id."""
name = name.strip()
if not name:
raise ValueError("an agent needs a name")
@@ -1897,23 +1924,36 @@ class Store:
prompts = check_prompts(list(prompts))
terms = None if fetch_terms is None else json.dumps(
normal_terms(list(fetch_terms)), ensure_ascii=False)
spoken = None if cv_languages is None else json.dumps(
[lang.model_dump() for lang in normal_languages(list(cv_languages))])
fields = (name, json.dumps([g.model_dump() for g in gates]),
(weights or Weights()).model_dump_json(),
(limits or Limits()).model_dump_json(), int(enabled), terms)
(limits or Limits()).model_dump_json(), int(enabled), terms,
None if cv_text is None else cv_text.strip(), spoken,
None if avoid_text is None else avoid_text.strip(),
None if must_text is None else must_text.strip())
try:
with self.transaction() as conn:
if agent_id is None:
agent_id = conn.execute(
"""INSERT INTO agents (name, gates, weights, limits, enabled,
fetch_terms, created_at)
VALUES (?,?,?,?,?,COALESCE(?, '[]'),?)""",
fetch_terms, cv_text, cv_languages, avoid_text,
must_text, created_at)
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),?)""",
(*fields, self.now())).lastrowid
elif not conn.execute(
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
fetch_terms=COALESCE(?, fetch_terms)
fetch_terms=COALESCE(?, fetch_terms),
cv_text=COALESCE(?, cv_text),
cv_languages=COALESCE(?, cv_languages),
avoid_text=COALESCE(?, avoid_text),
must_text=COALESCE(?, must_text)
WHERE id=?""",
(*fields, agent_id)).rowcount:
raise KeyError(agent_id)
conn.execute("UPDATE role_judgements SET tries = 0 "
"WHERE agent_id = ? AND tries > 0", (agent_id,))
cached = {r["text"]: (r["vec_key"], r["vec"]) for r in conn.execute(
"SELECT text, vec_key, vec FROM agent_prompts WHERE agent_id=?",
(agent_id,))}
+2 -2
View File
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27]
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27]
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28]
# The tag of 21 is All countries and All roles since 24.
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
+238
View File
@@ -0,0 +1,238 @@
"""The CV match's fields: an agent's CV, languages, avoid and must texts,
migration 28, the deadline from host and model, the languages extract reads
with a level, and the language flag. Nothing here reaches a model."""
from __future__ import annotations
import json
import sqlite3
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
from pydantic import ValidationError
from app import prep
from app.config import Config
from app.jobs import extract
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag
from app.storage import Store
from desk import modeltask
from desk.models import Item
from desk.tools import agent
from prepped import upsert
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def posting(source_id: str, body: str = "We build things.", **raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title="Head of Data", body=body,
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
item.source = "search"
return item
# ------------------------------------------------------------------ agents
def test_an_agent_keeps_its_cv_fields_when_a_save_leaves_them_out(store: Store) -> None:
made = store.save_agent("Data", "data leadership", cv_text=" VP Data. ",
cv_languages=[CvLanguage(language="DA", level="native"),
CvLanguage(language="en", level="fluent"),
CvLanguage(language="da", level="basic")],
avoid_text="Pure reporting.", must_text="Permanent contract.")
assert made.cv_text == "VP Data."
assert [(lang.language, lang.level) for lang in made.cv_languages] == \
[("da", "native"), ("en", "fluent")]
assert (made.avoid_text, made.must_text) == ("Pure reporting.", "Permanent contract.")
kept = store.save_agent("Data", "data leadership", agent_id=made.id)
assert (kept.cv_text, kept.avoid_text, kept.must_text) == \
("VP Data.", "Pure reporting.", "Permanent contract.")
assert len(kept.cv_languages) == 2
cleared = store.save_agent("Data", "data leadership", agent_id=made.id, cv_text="",
cv_languages=[], avoid_text="", must_text="")
assert (cleared.cv_text, cleared.cv_languages, cleared.avoid_text, cleared.must_text) \
== ("", [], "", "")
def test_a_cv_language_is_a_code_and_a_level_of_the_ladder() -> None:
with pytest.raises(ValidationError):
CvLanguage(language="Danish", level="native")
with pytest.raises(ValidationError):
CvLanguage(language="da", level="good")
def test_new_agents_have_judge_limits_by_default(store: Store) -> None:
made = store.save_agent("Data")
assert (made.limits.judge_top, made.limits.judge_min) == (150, 0.55)
# --------------------------------------------------------------- migration
def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path: Path) -> None:
path = tmp_path / "v27.db"
store = Store(path)
store.save_agent("Data", "data leadership")
store.upsert_role(posting("a", applicationDeadline="2026-10-18T00:00:00"))
store.upsert_role(posting("b"))
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_deadline")
conn.execute("DROP INDEX idx_role_judgements_agent")
conn.execute("DROP TABLE role_judgements")
for column in ("languages_required", "deadline_host", "deadline_model", "deadline"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
for column in ("cv_text", "cv_languages", "avoid_text", "must_text"):
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
conn.execute("PRAGMA user_version = 27")
migrated = Store(path)
assert migrated.migrations_applied == [28]
rows = {r["source_id"]: r for r in migrated._rows(
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
assert rows["a"]["languages_required"] is None and rows["b"]["deadline_host"] == ""
[data] = migrated.agents()
assert (data.cv_text, data.cv_languages, data.avoid_text, data.must_text) == \
("", [], "", "")
prep.run(migrated, remap=True, dupes=False)
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 27")
assert Store(path).migrations_applied == [28]
# ---------------------------------------------------------------- deadline
def test_iso_day_reads_a_date_or_a_timestamp() -> None:
assert iso_day("2026-10-18T00:00:00") == "2026-10-18"
assert iso_day("2026-10-18") == "2026-10-18"
assert iso_day("18.10.2026") == iso_day("2026-13-01") == iso_day(None) == ""
def test_the_deadline_is_the_hosts_first_then_the_models(store: Store) -> None:
host = posting("host", applicationDeadline="2026-10-18T00:00:00")
model = posting("model")
upsert(store, host)
upsert(store, model)
store.set_extracted(host.uid, {"deadline": "2026-11-01"})
store.set_extracted(model.uid, {"deadline": "2026-11-01"})
prep.run_roles(store, [host.uid, model.uid])
assert store.role(host.uid).deadline == "2026-10-18"
assert store.role(model.uid).deadline == "2026-11-01"
# A pull whose host no longer states one leaves the model's.
upsert(store, posting("host"))
assert store.role(host.uid).deadline == "2026-11-01"
# ----------------------------------------------------------------- extract
class Model:
def __init__(self, reply: Any) -> None:
self.reply = reply
self.asked: list[list[dict[str, Any]]] = []
def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply:
roles = kwargs["question"]["roles"]
self.asked.append(roles)
return agent.Reply(data={"roles": self.reply(roles)}, backend="api")
def answer(uid: str, **fields: Any) -> dict[str, Any]:
return {"uid": uid, "place": "", "country": "", "remote": "", "remote_where": [],
"seniority": "director", "salary_min": None, "salary_max": None,
"currency": "", "function": "", "permit": "", "languages_required": [],
"deadline": "", **fields}
def test_extract_reads_levels_kinds_and_the_deadline(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1")
upsert(store, item)
store.replace_scores(store.save_agent("Data", "data").id, [(item.uid, 0.9, "")])
model = Model(lambda roles: [answer(r["uid"], deadline="2026-10-30", languages_required=[
{"language": "DA", "level": "fluent", "kind": "required"},
{"language": "en", "level": "", "kind": "required"},
{"language": "sv", "level": "excellent", "kind": "plus"},
{"language": "da", "level": "basic", "kind": "plus"},
{"language": "danish", "level": "", "kind": "required"}]) for r in roles])
monkeypatch.setattr(agent, "backend", lambda settings: "api")
monkeypatch.setattr(agent, "ask_json", model)
config = Config(database=str(tmp_path / "roledesk.db"), extract={"enabled": True})
modeltask.run(extract.TASK, config, store)
role = store.role(item.uid)
assert role.languages == ["da", "en"]
assert [lang.model_dump() for lang in role.languages_required] == [
{"language": "da", "level": "fluent", "kind": "required"},
{"language": "en", "level": "", "kind": "required"},
{"language": "sv", "level": "", "kind": "plus"}]
assert role.deadline == "2026-10-30"
assert model.asked[0][0]["posted"] == role.posted_at[:10]
assert store.extract_pending() == 0
def test_a_role_extracted_before_is_read_once_more_for_the_new_fields(store: Store) -> None:
item = posting("1")
upsert(store, item)
store.replace_scores(store.save_agent("Data", "data").id, [(item.uid, 0.9, "")])
store.set_extracted(item.uid, {"seniority": "director"})
assert store.extract_pending() == 0
with store.transaction() as conn:
conn.execute("UPDATE roles SET languages_required = NULL")
assert store.extract_pending() == 1
def test_the_excerpt_keeps_the_opening_and_the_closing_lines() -> None:
short = "word " * 100
assert extract.excerpt(short) == short.strip()
long = "A" * 3000 + " " + "Z" * 3000
cut = extract.excerpt(long)
assert cut.startswith("A" * extract.HEAD) and cut.endswith("Z" * extract.TAIL)
assert len(cut) == extract.HEAD + extract.TAIL + 3
# ------------------------------------------------------------------- flags
def langs(*pairs: tuple[str, str]) -> list[CvLanguage]:
return [CvLanguage(language=code, level=level) for code, level in pairs]
def asks(*triples: tuple[str, str, str]) -> list[RoleLanguage]:
return [RoleLanguage(language=code, level=level, kind=kind) for code, level, kind in triples]
def test_the_language_flag() -> None:
mine = langs(("da", "native"), ("en", "fluent"), ("de", "professional"))
assert language_flag(asks(("sv", "fluent", "required")), mine) == \
("missing", "requires Swedish (fluent); yours: none")
assert language_flag(asks(("de", "fluent", "required")), mine) == \
("below", "requires German (fluent); yours: professional")
assert language_flag(asks(("de", "", "required"), ("da", "", "plus")), mine) == \
("plus", "Danish is a plus; yours: native")
assert language_flag(asks(("en", "fluent", "required"), ("fr", "", "plus")), mine) == \
("", "")
assert language_flag(asks(("sv", "fluent", "required")), []) == ("", "")
def test_the_cv_score_weighs_skills_and_career_above_experience() -> None:
assert cv_score(100, 100, 100) == 100.0
assert cv_score(80, 60, 40) == round((2400 + 1500 + 1200) / 85, 1)
def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
item = posting("1")
upsert(store, item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET languages_required = ? WHERE uid = ?",
(json.dumps([{"language": "da", "level": "fluent", "kind": "required"},
{"language": "en", "level": "superb"}]), item.uid))
assert [lang.language for lang in store.role(item.uid).languages_required] == ["da"]
+1 -1
View File
@@ -261,7 +261,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+3 -2
View File
@@ -32,7 +32,7 @@ def answer(uid: str, **fields: Any) -> dict[str, Any]:
empty: dict[str, Any] = {
"place": "", "country": "", "remote": "", "seniority": "", "salary_min": None,
"salary_max": None, "currency": "", "function": "", "permit": "",
"languages_required": []}
"languages_required": [], "deadline": ""}
return {"uid": uid, **empty, **fields}
@@ -139,7 +139,8 @@ def test_closed_and_answered_roles_are_not_offered(
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
"WHERE source_id = 'closed'")
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
"seniority_model = 'senior', remote_where_model = '[]' "
"seniority_model = 'senior', remote_where_model = '[]', "
"languages_required = '[]' "
"WHERE source_id = 'done'")
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
+1 -1
View File
@@ -434,7 +434,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
conn.execute("PRAGMA user_version = 22")
migrated = Store(path)
assert migrated.migrations_applied == [23, 24, 25, 26, 27]
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28]
assert migrated.agents()[0].fetch_terms == []
+1 -1
View File
@@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
conn.execute("PRAGMA user_version = 23")
migrated = Store(path)
assert migrated.migrations_applied == [24, 25, 26, 27]
assert migrated.migrations_applied == [24, 25, 26, 27, 28]
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
+1 -1
View File
@@ -316,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27]
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
+2 -2
View File
@@ -530,7 +530,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
migrated = Store(path)
assert migrated.migrations_applied == [27]
assert migrated.migrations_applied == [27, 28]
assert migrated.gazetteer().anywhere is not None
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
"WHERE uid = ?", (board,))
@@ -540,7 +540,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
assert area(migrated, search.uid) == ("", "")
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 26")
assert Store(path).migrations_applied == [27]
assert Store(path).migrations_applied == [27, 28]
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
+10 -10
View File
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 27
assert LATEST == 28
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -173,7 +173,7 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
assert Store(old).migrations_applied == list(range(2, LATEST + 1))
# ALTER appends, so the order differs; every read is by column name.
for table in ("roles", "agents", "agent_prompts", "scores", "vectors", "companies",
"role_tags", "agent_companies", "role_votes"):
"role_tags", "agent_companies", "role_votes", "role_judgements"):
assert sorted(_columns(old, table)) == sorted(_columns(fresh, table))
@@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -688,7 +688,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27]
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -730,7 +730,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
migrated = Store(path)
assert migrated.migrations_applied == [25, 26, 27]
assert migrated.migrations_applied == [25, 26, 27, 28]
with sqlite3.connect(path) as conn:
found = {sid: rest for sid, *rest in conn.execute(
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
@@ -748,12 +748,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
migrated = Store(path)
assert migrated.migrations_applied == [26, 27]
assert migrated.migrations_applied == [26, 27, 28]
role = migrated.role(posting("a").uid)
assert role is not None and role.posting_url == ""
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 25") # the column is there already
assert Store(path).migrations_applied == [26, 27]
assert Store(path).migrations_applied == [26, 27, 28]
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
+6 -3
View File
@@ -1182,8 +1182,10 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No
0..1 is refused."""
default = client.post("/api/agents", json={"name": "Wide"}).json()
assert default["limits"] == {"extract_top": 500, "extract_min": 0.55,
"summarise_top": 30, "summarise_min": 0.65}
narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7}
"summarise_top": 30, "summarise_min": 0.65,
"judge_top": 150, "judge_min": 0.55}
narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7,
"judge_top": 20, "judge_min": 0.6}
saved = client.post("/api/agents", json={"name": "Narrow", "limits": narrow})
assert saved.status_code == 200 and saved.json()["limits"] == narrow
assert [a["limits"]["extract_top"] for a in client.get("/api/agents").json()] == [50, 500]
@@ -1227,7 +1229,8 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
"seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state",
"original_url", "posting_url", "posted_at", "score", "gated_reason", "prompt", "vote",
"voted_at", "read_at", "tags", "excerpt", "copies", "remote_where",
"voted_at", "read_at", "tags", "excerpt", "copies", "remote_where", "deadline",
"lang_flag", "lang_note", "cv_score",
}
assert row["uid"] == uid
assert row["company"] == "Acme" and row["place"] == "Berlin, Germany"