Merge branch 'wp4-prompts'
This commit is contained in:
commit
7939b99a5c
11 files changed
+658
-133
No files matched your search
+67
-46
@@ -14,10 +14,11 @@ A gate rejects on evidence, never on absence: a role whose column is empty
|
||||
is not gated by it. `extract` is what fills an empty column, and it reads
|
||||
only the roles that rank, so gating on an empty column would keep those
|
||||
roles out of extract's reach for good.
|
||||
fit cos(role vector, intent vector) as a level: the embedding
|
||||
fit cos(role vector, prompt vector) as a level: the embedding
|
||||
model's calibrated `lo` is 0 and `hi` 1, clipped outside
|
||||
(see `scale`), so a role's fit does not depend on what
|
||||
else is open
|
||||
else is open; with several prompts, the best prompt's
|
||||
level, and the score names that prompt
|
||||
recency 0.5 ** (days since posted_at / 14)
|
||||
affinity (the agent's affinity for the company + 1) / 3
|
||||
taste cos(role vector, liked direction), scaled like fit with the
|
||||
@@ -32,10 +33,10 @@ company (`affinity = 2`) skips the gates and is floored at 0.9; a hidden one
|
||||
(`-1`) is not scored at all. A role with no vector is scored without `fit`
|
||||
and `taste`, unless `fit` is all there is, when it is gated as "no vector".
|
||||
An agent with no votes scores every role without `taste`; the direction is
|
||||
newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no intent
|
||||
has no `fit` either: its scores are not a match, so it is left out of what
|
||||
extract and summarise read (`Store._top_scope`), and a run names such agents
|
||||
in one notice.
|
||||
newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no prompt
|
||||
that has text has no `fit` either: its scores are not a match, so it is left
|
||||
out of what extract and summarise read (`Store._top_scope`), and a run names
|
||||
such agents in one notice.
|
||||
|
||||
Recomputed in full on every run, and each agent's rows are replaced in one
|
||||
transaction: nothing is incremental, so nothing can drift.
|
||||
@@ -176,7 +177,7 @@ def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
|
||||
missing: str = "no vector") -> tuple[float, str]:
|
||||
"""One role for one agent: `(score, gated_reason)`. `fit` and `taste` are
|
||||
already scaled to 0..1 (see `scale`), or None when there is nothing to
|
||||
measure: no role vector, no intent vector, no votes. A None term leaves
|
||||
measure: no role vector, no prompt vector, no votes. A None term leaves
|
||||
the denominator; `missing` says why fit is out."""
|
||||
favourite = candidate.affinity == 2
|
||||
if failed := failed_gates(candidate, gates):
|
||||
@@ -207,8 +208,8 @@ def intent_key(model: str, intent: str) -> str:
|
||||
|
||||
|
||||
class _Encoder:
|
||||
"""The embedder, loaded on first use only: an agent whose intent vector
|
||||
is cached costs no model load at all."""
|
||||
"""The embedder, loaded on first use only: an agent whose prompt vectors
|
||||
are cached costs no model load at all."""
|
||||
|
||||
def __init__(self, model: str, context: RunContext) -> None:
|
||||
self.model = model
|
||||
@@ -229,66 +230,85 @@ class _Encoder:
|
||||
return np.asarray(self._embedder.encode([text])[0])
|
||||
|
||||
|
||||
def _intent(store: Store, agent: Agent, model: str,
|
||||
encoder: _Encoder) -> np.ndarray | None:
|
||||
if not agent.intent.strip():
|
||||
return None
|
||||
key = intent_key(model, agent.intent)
|
||||
cached = store.intent_vector(agent.id, key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
vec = encoder.encode(embed.intent_text(model, agent.intent))
|
||||
if vec is not None:
|
||||
store.set_intent_vector(agent.id, key, vec)
|
||||
return vec
|
||||
def _prompts(store: Store, agent: Agent, model: str,
|
||||
encoder: _Encoder) -> list[tuple[str, np.ndarray]]:
|
||||
"""`(name, vector)` for each of the agent's prompts that has text and
|
||||
could be embedded, in order. Each vector is cached under its own key, so
|
||||
an edit re-embeds only the prompt it changed."""
|
||||
found = []
|
||||
for prompt in agent.prompts:
|
||||
if not prompt.text.strip():
|
||||
continue
|
||||
key = intent_key(model, prompt.text)
|
||||
vec = store.prompt_vector(agent.id, prompt.name, key)
|
||||
if vec is None:
|
||||
vec = encoder.encode(embed.intent_text(model, prompt.text))
|
||||
if vec is None:
|
||||
continue
|
||||
store.set_prompt_vector(agent.id, prompt.name, key, vec)
|
||||
found.append((prompt.name, vec))
|
||||
return found
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- the job
|
||||
|
||||
def no_intent(names: Sequence[str]) -> Notice:
|
||||
"""The notice for enabled agents without an intent: their scores have no
|
||||
fit, so `Store._top_scope` keeps them from feeding extract and summarise."""
|
||||
"""The notice for enabled agents without a prompt that has text: their
|
||||
scores have no fit, so `Store._top_scope` keeps them from feeding extract
|
||||
and summarise. The code keeps its name from when an agent had one intent."""
|
||||
return Notice(
|
||||
code="score.no_intent", level="warning", kind="expected", key="score",
|
||||
text=f"No intent, so ranked without fit and not read by extract or "
|
||||
f"summarise: {', '.join(names)}.",
|
||||
fix="Give the agent an intent under Settings, Search agents.",
|
||||
text=f"No prompt with text, so ranked without fit and not read by extract "
|
||||
f"or summarise: {', '.join(names)}.",
|
||||
fix="Give the agent a prompt under Settings, Search agents.",
|
||||
where="app.jobs.score")
|
||||
|
||||
|
||||
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||
intent: np.ndarray | None, now: datetime,
|
||||
prompts: Sequence[tuple[str, np.ndarray]], now: datetime,
|
||||
calibration: Calibration | None = None) -> dict[str, int]:
|
||||
"""Two passes: the cosines of the roles this agent scores, to its intent
|
||||
and to its taste direction, then every score with both scaled by the
|
||||
model's `calibration`, or across the run without one. A gated role is
|
||||
left out of the cosines, so under the fallback a role that fails a
|
||||
require gate cannot stretch the range the ranked roles share. The voted
|
||||
roles are scored like any other; a liked one sits near the direction it
|
||||
helped make, which is what a like should do."""
|
||||
"""Two passes: the cosines of the roles this agent scores, to each of its
|
||||
`prompts` and to its taste direction, then every score with both scaled
|
||||
by the model's `calibration`, or across the run without one. A role's fit
|
||||
is the best of its prompts' levels, each scaled on its own, and the score
|
||||
keeps that prompt's name; the first prompt wins a tie. Under the min-max
|
||||
fallback each prompt is ranged over the run on its own too, so every
|
||||
prompt's closest role gets 1 and one prompt equals the old single intent.
|
||||
A gated role is left out of the cosines, so under the fallback a role
|
||||
that fails a require gate cannot stretch the range the ranked roles
|
||||
share. The voted roles are scored like any other; a liked one sits near
|
||||
the direction it helped make, which is what a like should do."""
|
||||
candidates = store.score_candidates(agent.id, agent.gates)
|
||||
direction = taste_direction(vectors, store.votes(agent.id))
|
||||
cosines: dict[str, float] = {}
|
||||
cosines: dict[str, dict[str, float]] = {name: {} for name, _ in prompts}
|
||||
tastes: dict[str, float] = {}
|
||||
for candidate in candidates:
|
||||
role_vec = vectors.get(candidate.uid)
|
||||
if role_vec is None or failed_gates(candidate, agent.gates):
|
||||
continue
|
||||
if intent is not None:
|
||||
cosines[candidate.uid] = cosine(role_vec, intent)
|
||||
for name, prompt_vec in prompts:
|
||||
cosines[name][candidate.uid] = cosine(role_vec, prompt_vec)
|
||||
if direction is not None:
|
||||
tastes[candidate.uid] = cosine(role_vec, direction)
|
||||
fits = scale(cosines, calibration.fit if calibration else None)
|
||||
fits: dict[str, float] = {}
|
||||
best: dict[str, str] = {}
|
||||
for name, levels in cosines.items():
|
||||
for uid, value in scale(levels, calibration.fit if calibration else None).items():
|
||||
if uid not in fits or value > fits[uid]:
|
||||
fits[uid], best[uid] = value, name
|
||||
tastes = scale(tastes, calibration.taste if calibration else None)
|
||||
rows: list[tuple[str, float, str]] = []
|
||||
rows: list[tuple[str, float, str, str]] = []
|
||||
gated = 0
|
||||
for candidate in candidates:
|
||||
fit = fits.get(candidate.uid)
|
||||
value, reason = score_one(
|
||||
candidate, agent.gates, agent.weights, fits.get(candidate.uid), now,
|
||||
candidate, agent.gates, agent.weights, fit, now,
|
||||
taste=tastes.get(candidate.uid),
|
||||
missing="no vector" if intent is not None else "no intent vector")
|
||||
missing="no vector" if prompts else "no prompt vector")
|
||||
gated += bool(reason)
|
||||
rows.append((candidate.uid, round(value, 6), reason))
|
||||
counted = fit is not None and agent.weights.fit and not reason
|
||||
rows.append((candidate.uid, round(value, 6), reason,
|
||||
best[candidate.uid] if counted else ""))
|
||||
store.replace_scores(agent.id, rows)
|
||||
return {"scored": len(rows), "gated": gated}
|
||||
|
||||
@@ -307,14 +327,15 @@ def run_score(config: Config, store: Store, *,
|
||||
log.warning("no calibration for %s: fit and taste are min-max scaled across "
|
||||
"each agent's roles in this run, so they rank rather than state a "
|
||||
"level; add it under embed.calibration", model)
|
||||
if blind := [agent.name for agent in agents if not agent.intent.strip()]:
|
||||
if blind := [agent.name for agent in agents
|
||||
if not any(p.text.strip() for p in agent.prompts)]:
|
||||
context.notice(no_intent(blind))
|
||||
scored = gated = 0
|
||||
for n, agent in enumerate(agents):
|
||||
context.check()
|
||||
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
|
||||
counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now,
|
||||
calibration)
|
||||
counts = score_agent(store, agent, vectors, _prompts(store, agent, model, encoder),
|
||||
now, calibration)
|
||||
scored += counts["scored"]
|
||||
gated += counts["gated"]
|
||||
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])
|
||||
@@ -335,7 +356,7 @@ def seed_agents(config: Config, store: Store) -> list[str]:
|
||||
for seed in config.agents:
|
||||
try:
|
||||
store.save_agent(seed.name, seed.intent, seed.gates, seed.weights,
|
||||
seed.enabled, limits=seed.limits)
|
||||
seed.enabled, limits=seed.limits, prompts=seed.prompts)
|
||||
except ValueError as exc:
|
||||
log.warning("agents: %s", exc)
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user