Score each role's closeness to the agent's CV, let the judge also read the 50 closest the score leaves out, and order CV match's unjudged roles by it
The CV is cut into chunks inside e5's window, embedded like a prompt and cached per agent under a hash of the model, chunking and text. A role's cv_fit (scores.cv_fit, migration 31) is its best chunk cosine, scaled with its own calibration band: fit's clipped 88 of agent 2's ungated roles to 1. It is no term of the Agent match score. judge_cv_top (default 50) sits beside judge_top and judge_min on the agents page and card. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b481cb5692
commit
c23594507a
23 files changed
+603
-85
No files matched your search
@@ -77,6 +77,62 @@ def intent_text(model: str, intent: str) -> str:
|
||||
return f"query: {intent}" if wants_e5_prefix(model) else intent
|
||||
|
||||
|
||||
#: The most of a CV one vector reads. multilingual-e5-small reads 512 tokens
|
||||
#: and cuts the rest; on the owner's English CV a paragraph runs about four
|
||||
#: characters a token (1,896 characters were 471 tokens with "query: "), so
|
||||
#: 1,200 characters stay inside the window with room for a denser language.
|
||||
CV_CHUNK_CHARS = 1200
|
||||
|
||||
#: The version of `cv_chunks`, part of the CV vectors' cache key
|
||||
#: (`score.cv_key`): a change to how a CV is cut embeds every CV again.
|
||||
CV_CHUNKING = 1
|
||||
|
||||
#: Where a long paragraph is cut, coarsest first, and what joins its pieces
|
||||
#: again: lines, then sentences.
|
||||
_CUTS = ((re.compile(r"\n+"), "\n"), (re.compile(r"(?<=[.!?;])\s+"), " "))
|
||||
|
||||
|
||||
def _pieces(text: str, limit: int) -> list[str]:
|
||||
"""`text` cut to pieces of at most `limit` characters: by line, then by
|
||||
sentence, then at the last space before the limit, else at the limit."""
|
||||
if len(text) <= limit:
|
||||
return [text]
|
||||
for pattern, glue in _CUTS:
|
||||
parts = [p.strip() for p in pattern.split(text) if p.strip()]
|
||||
if len(parts) > 1:
|
||||
return _merge([piece for part in parts for piece in _pieces(part, limit)],
|
||||
limit, glue)
|
||||
cut = text.rfind(" ", 0, limit + 1)
|
||||
cut = cut if cut > 0 else limit
|
||||
return [text[:cut].strip(), *_pieces(text[cut:].strip(), limit)]
|
||||
|
||||
|
||||
def _merge(pieces: list[str], limit: int, glue: str) -> list[str]:
|
||||
"""Neighbouring pieces joined with `glue` while the join fits `limit`."""
|
||||
merged: list[str] = []
|
||||
for piece in pieces:
|
||||
if merged and len(merged[-1]) + len(glue) + len(piece) <= limit:
|
||||
merged[-1] += glue + piece
|
||||
else:
|
||||
merged.append(piece)
|
||||
return merged
|
||||
|
||||
|
||||
def cv_chunks(cv_text: str, limit: int | None = None) -> list[str]:
|
||||
"""A CV as the texts its vectors are made from: its paragraphs (blank
|
||||
lines apart), a longer one cut by line, then by sentence, and neighbours
|
||||
joined again while they fit `limit` (`CV_CHUNK_CHARS`), so a heading
|
||||
travels with what follows it and no chunk runs past what the model
|
||||
reads."""
|
||||
limit = limit or CV_CHUNK_CHARS
|
||||
pieces = []
|
||||
for paragraph in re.split(r"\n\s*\n", cv_text):
|
||||
lines = "\n".join(" ".join(line.split()) for line in paragraph.splitlines()).strip()
|
||||
if lines:
|
||||
pieces += _pieces(lines, limit)
|
||||
return _merge(pieces, limit, "\n\n")
|
||||
|
||||
|
||||
def load_embedder(model: str) -> embedding.Embedder:
|
||||
"""The encoder. A seam of its own so a test hands in a fake and nothing
|
||||
downloads a model; `score` loads through it too."""
|
||||
|
||||
+4
-3
@@ -14,9 +14,10 @@ is guessed from ad text, the second is what the gates do.
|
||||
|
||||
The work is pairs, `(agent, role)`: an enabled agent with a CV, and a role
|
||||
among its best ungated scores as deep and as low as its own `limits` say
|
||||
(`judge_top`, `judge_min`) with some text, that has no judgement yet, or one
|
||||
read from another CV, prompts, like, avoid or must text or role text than today's
|
||||
(`input_key`). A pair is named by the composite key `"<agent_id>:<uid>"`,
|
||||
(`judge_top`, `judge_min`), or among the `judge_cv_top` ungated roles closest
|
||||
to its CV (`scores.cv_fit`) that those leave out, with some text, that has
|
||||
no judgement yet, or one read from another CV, prompts, like, avoid or must
|
||||
text or role text than today's (`input_key`). A pair is named by the composite key `"<agent_id>:<uid>"`,
|
||||
which is what `desk.modeltask` hands round as a uid, so `desk/` is untouched.
|
||||
A batch holds one agent's pairs, since the profile is in the system prompt;
|
||||
the finder cuts per agent (`_finder`) and the job runs one call per agent it
|
||||
|
||||
+61
-6
@@ -29,6 +29,12 @@ roles out of extract's reach for good.
|
||||
over this agent's votes; with only likes it is the liked mean,
|
||||
with only passes the negated passed mean; no votes, no term
|
||||
|
||||
Beside the score, for an agent with a CV, `cv_fit`: how close the role is to
|
||||
the CV, the best cosine between the role and the CV's chunks
|
||||
(`embed.cv_chunks`, embedded as the prompts are), scaled like fit. It is no
|
||||
term of the score; the judge reads the closest roles the score leaves out
|
||||
(`Store.judge_rows`) and CV match orders the roles not judged yet by it.
|
||||
|
||||
Every term is 0..1. The score is the weighted sum over the sum of the weights
|
||||
(the agent's four and its prefer gates'), so it stays 0..1. A favourite
|
||||
company (`affinity = 2`) skips the `any` group but not the `require` or
|
||||
@@ -208,6 +214,21 @@ def intent_key(model: str, intent: str) -> str:
|
||||
return hashlib.sha1(f"{model}\n{intent}".encode()).hexdigest()
|
||||
|
||||
|
||||
def cv_key(model: str, cv_text: str) -> str:
|
||||
"""What the CV's vectors are cached under: the model, how the CV is cut
|
||||
(`embed.CV_CHUNKING`) and the text."""
|
||||
return hashlib.sha1(f"{model}\n{embed.CV_CHUNKING}\n{cv_text}".encode()).hexdigest()
|
||||
|
||||
|
||||
def closeness(role: np.ndarray, chunks: np.ndarray) -> float:
|
||||
"""The best raw cosine between a role and the CV's chunks, -1..1, with
|
||||
`chunks` one unit-length row each (`_cv`); 0 for a zero role vector."""
|
||||
norm = float(np.linalg.norm(role))
|
||||
if norm == 0 or not len(chunks):
|
||||
return 0.0
|
||||
return float(np.max(chunks @ role)) / norm
|
||||
|
||||
|
||||
class _Encoder:
|
||||
"""The embedder, loaded on first use only: an agent whose prompt vectors
|
||||
are cached costs no model load at all."""
|
||||
@@ -219,6 +240,10 @@ class _Encoder:
|
||||
self.failed = False
|
||||
|
||||
def encode(self, text: str) -> np.ndarray | None:
|
||||
found = self.encode_many([text])
|
||||
return None if found is None else found[0]
|
||||
|
||||
def encode_many(self, texts: Sequence[str]) -> np.ndarray | None:
|
||||
if self.failed:
|
||||
return None
|
||||
if self._embedder is None:
|
||||
@@ -228,7 +253,7 @@ class _Encoder:
|
||||
self.failed = True
|
||||
self.context.notice(embed.unavailable(exc))
|
||||
return None
|
||||
return np.asarray(self._embedder.encode([text])[0])
|
||||
return np.asarray(self._embedder.encode(list(texts)))
|
||||
|
||||
|
||||
def _prompts(store: Store, agent: Agent, model: str,
|
||||
@@ -251,6 +276,27 @@ def _prompts(store: Store, agent: Agent, model: str,
|
||||
return found
|
||||
|
||||
|
||||
def _cv(store: Store, agent: Agent, model: str, encoder: _Encoder) -> np.ndarray | None:
|
||||
"""The agent's CV as one unit-length row per chunk (`embed.cv_chunks`),
|
||||
each encoded as the prompts are ("query: " for an e5 model), or None
|
||||
without a CV or an embedder. Cached under `cv_key`, so a run embeds the
|
||||
CV only after it changed."""
|
||||
if not agent.cv_text.strip():
|
||||
return None
|
||||
key = cv_key(model, agent.cv_text)
|
||||
found = store.cv_vectors(agent.id, key)
|
||||
if found is None:
|
||||
chunks = embed.cv_chunks(agent.cv_text)
|
||||
made = encoder.encode_many([embed.intent_text(model, c) for c in chunks])
|
||||
if made is None or not len(made):
|
||||
return None
|
||||
found = list(made)
|
||||
store.set_cv_vectors(agent.id, key, found)
|
||||
rows = np.vstack(found).astype(np.float64)
|
||||
norms = np.linalg.norm(rows, axis=1, keepdims=True)
|
||||
return rows / np.where(norms == 0, 1.0, norms)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- the job
|
||||
|
||||
def no_intent(names: Sequence[str]) -> Notice:
|
||||
@@ -267,7 +313,8 @@ def no_intent(names: Sequence[str]) -> Notice:
|
||||
|
||||
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||
prompts: Sequence[tuple[str, np.ndarray]], now: datetime,
|
||||
calibration: Calibration | None = None) -> dict[str, int]:
|
||||
calibration: Calibration | None = None,
|
||||
cv: np.ndarray | None = None) -> dict[str, int]:
|
||||
"""Two passes: the cosines of the roles this agent scores, to each of its
|
||||
`prompts` and to its taste direction, then every score with both scaled
|
||||
by the model's `calibration`, or across the run without one. A role's fit
|
||||
@@ -278,11 +325,14 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||
A gated role is left out of the cosines, so under the fallback a role
|
||||
that fails a require gate cannot stretch the range the ranked roles
|
||||
share. The voted roles are scored like any other; a liked one sits near
|
||||
the direction it helped make, which is what a like should do."""
|
||||
the direction it helped make, which is what a like should do. With `cv`
|
||||
(`_cv`), each of those roles also gets its `cv_fit`, its `closeness` to
|
||||
the CV scaled like fit, beside the score and not in it."""
|
||||
candidates = store.score_candidates(agent.id, agent.gates)
|
||||
direction = taste_direction(vectors, store.votes(agent.id))
|
||||
cosines: dict[str, dict[str, float]] = {name: {} for name, _ in prompts}
|
||||
tastes: dict[str, float] = {}
|
||||
near: dict[str, float] = {}
|
||||
for candidate in candidates:
|
||||
role_vec = vectors.get(candidate.uid)
|
||||
if role_vec is None or failed_gates(candidate, agent.gates):
|
||||
@@ -291,6 +341,8 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||
cosines[name][candidate.uid] = cosine(role_vec, prompt_vec)
|
||||
if direction is not None:
|
||||
tastes[candidate.uid] = cosine(role_vec, direction)
|
||||
if cv is not None:
|
||||
near[candidate.uid] = closeness(role_vec, cv)
|
||||
fits: dict[str, float] = {}
|
||||
best: dict[str, str] = {}
|
||||
for name, levels in cosines.items():
|
||||
@@ -298,7 +350,8 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||
if uid not in fits or value > fits[uid]:
|
||||
fits[uid], best[uid] = value, name
|
||||
tastes = scale(tastes, calibration.taste if calibration else None)
|
||||
rows: list[tuple[str, float, str, str]] = []
|
||||
near = scale(near, calibration.cv_band if calibration else None)
|
||||
rows: list[tuple[str, float, str, str, float | None]] = []
|
||||
gated = 0
|
||||
for candidate in candidates:
|
||||
fit = fits.get(candidate.uid)
|
||||
@@ -308,8 +361,10 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||
missing="no vector" if prompts else "no prompt vector")
|
||||
gated += bool(reason)
|
||||
counted = fit is not None and agent.weights.fit and not reason
|
||||
cv_fit = near.get(candidate.uid)
|
||||
rows.append((candidate.uid, round(value, 6), reason,
|
||||
best[candidate.uid] if counted else ""))
|
||||
best[candidate.uid] if counted else "",
|
||||
None if cv_fit is None else round(cv_fit, 6)))
|
||||
store.replace_scores(agent.id, rows)
|
||||
return {"scored": len(rows), "gated": gated}
|
||||
|
||||
@@ -336,7 +391,7 @@ def run_score(config: Config, store: Store, *,
|
||||
context.check()
|
||||
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
|
||||
counts = score_agent(store, agent, vectors, _prompts(store, agent, model, encoder),
|
||||
now, calibration)
|
||||
now, calibration, cv=_cv(store, agent, model, encoder))
|
||||
scored += counts["scored"]
|
||||
gated += counts["gated"]
|
||||
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])
|
||||
|
||||
Reference in new issue
Block a user