Rank by the cut: fetch gate overrides in scoring, the cut per agent, stages read the cut, fetch text renamed from enrich, the feed kept to the fetch gate, and migration 32

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-10-04 15:11:43 +02:00
1 parent 060ccf66e6
commit 617356e60c
35 files changed
+1054 -751

No files matched your search

+101 -4
View File
@@ -385,10 +385,14 @@ CREATE TABLE IF NOT EXISTS role_votes (
PRIMARY KEY (agent_id, uid)
);
-- A role is scored per agent. A gated role keeps its row, with the reason.
-- prompt names the agent's prompt that gave the fit, '' when fit did not count.
-- cv_fit is how close the role is to the agent's CV, 0..1, NULL without a CV
-- or a vector and for a gated role; it is no term of score.
-- A role is scored per agent. A role the fetch gate rejects keeps its row,
-- with the reason. score is the final score, the feed's order. prompt names
-- the agent's prompt that gave the fit, '' when fit did not count. fit is the
-- prompt fit level and cv_fit how close the role is to the agent's CV, both
-- 0..1 before the affinity bonus, NULL without a prompt, a CV or a vector and
-- for a gated role. cut says why the role is in the agent's cut, the roles
-- that get fetch text, extract and judge: 'fit', 'cv', 'kept' (liked or
-- saved), or '' when it is not.
CREATE TABLE IF NOT EXISTS scores (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
@@ -397,9 +401,23 @@ CREATE TABLE IF NOT EXISTS scores (
scored_at TEXT NOT NULL,
prompt TEXT NOT NULL DEFAULT '',
cv_fit REAL,
fit REAL,
cut TEXT NOT NULL DEFAULT '',
PRIMARY KEY (uid, agent_id)
);
-- The feed's filter settings saved under a name, per agent. query is the
-- feed's query string without the agent. Names are unique per agent
-- regardless of case; a change is a new row.
CREATE TABLE IF NOT EXISTS saved_filters (
id INTEGER PRIMARY KEY,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
name TEXT NOT NULL,
query TEXT NOT NULL,
created_at TEXT NOT NULL,
UNIQUE (agent_id, name COLLATE NOCASE)
);
-- The vectors of an agent's CV, one per chunk in order, for scores.cv_fit.
-- vec_key is sha1 of the embedding model, the chunking and the CV text, the
-- same on every row of an agent, so an unchanged CV is not embedded again.
@@ -487,6 +505,9 @@ CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id
CREATE INDEX IF NOT EXISTS idx_application_events
ON application_events(application_id, at);
CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
-- The roles in an agent's cut, which fetch text, extract and judge read.
-- Partial: most roles are in no cut.
CREATE INDEX IF NOT EXISTS idx_scores_cut ON scores(agent_id, uid) WHERE cut <> '';
CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote);
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
@@ -731,6 +752,12 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
# missing. The next score fills them.
(31, "scores.cv_fit and agent_cv_vectors: how close each role is to the agent's CV",
lambda conn: _add_cv_fit(conn)),
# A Python step: gates and limits are JSON. The old limits were bars on
# the blended score and mean nothing on plain fit, so every agent takes
# the new defaults.
(32, "the cut: scores.fit and scores.cut, agents' limits reset to the cut's defaults, "
"prefer gates gone, saved_filters, and enrich renamed fetch text",
lambda conn: _add_cut(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -1094,6 +1121,76 @@ def _add_cv_fit(conn: sqlite3.Connection) -> None:
PRIMARY KEY (agent_id, position))""")
#: What migration 32 writes into every agent's `limits`: `models.Limits` as
#: it stood then, spelled out so a later change of the defaults does not
#: change what this migration did.
_V32_LIMITS = {"cut_fit_top": 500, "cut_fit_min": 0.6, "cut_cv_top": 150, "cut_cv_min": 0.6,
"summarise_top": 50}
_V32 = """
CREATE TABLE IF NOT EXISTS saved_filters (
id INTEGER PRIMARY KEY,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
name TEXT NOT NULL,
query TEXT NOT NULL,
created_at TEXT NOT NULL,
UNIQUE (agent_id, name COLLATE NOCASE)
);
INSERT OR IGNORE INTO state (key, value)
SELECT 'stage:fetch_text', value FROM state WHERE key = 'stage:enrich';
DELETE FROM state WHERE key = 'stage:enrich';
"""
def _add_cut(conn: sqlite3.Connection) -> None:
"""Migration 32's step. `scores.fit` and `scores.cut`, NULL and empty
until the next score; every agent's `limits` reset to `_V32_LIMITS`;
the `prefer` gates dropped from every agent's `gates` and `weight` from
the others; `saved_filters`; the switch `stage:enrich` renamed
`stage:fetch_text`; and the queued `enrich` and `enrich_batch` jobs
deleted, with a schedule of `enrich` firing `fetch_text`. Runs twice
without harm: limits are reset when the columns are added, and after
that only where they still hold a key the cut does not know."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(scores)")}
fresh = "cut" not in columns
if "fit" not in columns:
conn.execute("ALTER TABLE scores ADD COLUMN fit REAL")
if fresh:
conn.execute("ALTER TABLE scores ADD COLUMN cut TEXT NOT NULL DEFAULT ''")
reset = dropped = 0
for agent_id, raw_limits, raw_gates in conn.execute(
"SELECT id, limits, gates FROM agents").fetchall():
try:
limits = json.loads(raw_limits or "{}")
except ValueError:
limits = None
if fresh or not isinstance(limits, dict) or set(limits) - set(_V32_LIMITS):
conn.execute("UPDATE agents SET limits=? WHERE id=?",
(json.dumps(_V32_LIMITS), agent_id))
reset += 1
try:
gates = json.loads(raw_gates or "[]")
except ValueError:
continue
if not isinstance(gates, list):
continue
kept = [{k: v for k, v in g.items() if k != "weight"} for g in gates
if isinstance(g, dict) and g.get("mode") != "prefer"]
if kept != gates:
dropped += sum(1 for g in gates if isinstance(g, dict) and g.get("mode") == "prefer")
conn.execute("UPDATE agents SET gates=? WHERE id=?", (json.dumps(kept), agent_id))
for statement in _statements(_V32):
conn.execute(statement)
tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if "jobs" in tables:
conn.execute("DELETE FROM jobs WHERE state = 'queued' "
"AND kind IN ('enrich', 'enrich_batch')")
if "schedules" in tables:
conn.execute("UPDATE schedules SET kind = 'fetch_text' WHERE kind = 'enrich'")
log.info("migration 32: %d agents' limits reset to the cut's defaults, %d prefer gates "
"dropped", reset, dropped)
def _add_all_roles(conn: sqlite3.Connection) -> None:
"""Migration 21's step. The column; the tag for every company that is a
favourite of any agent, since favourite no longer widens what a pull
+167 -125
View File
@@ -25,7 +25,7 @@ from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
RoleCopy, RolePlace, RoleRow, ScoredRole, cv_languages as normal_languages, cv_score,
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
BROKEN_CAP, capped_career, language_flag, role_languages,
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
@@ -42,7 +42,7 @@ _RAW_TEXT = ("place", "currency")
#: The host's own words a reader hands on in `Item.raw`, stored as
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
#: rewritten every pull, as the host's level always was; the others are kept
#: filled, since a role's own page (`enrich`) may state what its listing
#: filled, since a role's own page (`fetch_text`) may state what its listing
#: does not.
_HOST_WORDS = ("seniority", "employment", "remote", "country")
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
@@ -78,6 +78,16 @@ _EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_whe
"OR languages_required IS NULL) "
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: A role in the cut of an enabled agent (`scores.cut`, see `app.jobs.score`),
#: as a `WHERE` on `r`: what fetch text and extract read.
_IN_CUT = ("r.uid IN (SELECT s.uid FROM scores s JOIN agents a ON a.id = s.agent_id "
"AND a.enabled = 1 WHERE s.cut <> '')")
#: The order within one agent's cut, over its scores as `s`: in by fit, best
#: fit first, then in by CV closeness, closest first, then kept.
_CUT_ORDER = ("CASE s.cut WHEN 'fit' THEN 0 WHEN 'cv' THEN 1 ELSE 2 END, "
"CASE s.cut WHEN 'fit' THEN s.fit WHEN 'cv' THEN s.cv_fit END DESC")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
VOTES = (-1, 0, 1, None)
@@ -1127,7 +1137,7 @@ class Store:
hide_muted: bool = False, include_read: bool = True,
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, pay: str | None = None,
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
company_keys: Sequence[str] = (), places: Sequence[str] = (),
seniority: Sequence[str] = (), seniority_unknown: bool = True,
pay_unknown: bool = False, by_cv: bool = False, closing: bool = False,
@@ -1172,7 +1182,9 @@ class Store:
set. The score is the stored one moved by the agent's vote
(`_VOTED_SCORE`), so a vote shows in the order at once and needs no
rescore. With `by_score` off the order is newest, with the votes and
no score.
no score; with `fetch_gate` as well, the newest order keeps to the
agent's fetch gate the same way, carrying `gated_reason`, so no order
shows a role the agent's gate rejects unless `gated`.
With `by_cv` as well, the roles the agent's `judge` read come first
by `cv_score`, then the others by how close they are to the CV
@@ -1191,6 +1203,7 @@ class Store:
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
fetch_gate=fetch_gate,
judged=judged, pay=pay, company_keys=company_keys, places=places,
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
closing=closing)
@@ -1202,7 +1215,9 @@ class Store:
if scored:
select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt, s.cv_fit"
order = (_by_cv(cv) if by_cv else "") + "score DESC, " + order
elif open_only:
elif agent is not None and fetch_gate:
select += ", s.gated_reason"
if not scored and open_only:
# Newest first, open only: idx_roles_posted covers this exactly
# (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
# LIMIT rather than sorting every open role first. Left to the
@@ -1241,7 +1256,7 @@ class Store:
hide_muted: bool = False, include_read: bool = True,
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, pay: str | None = None,
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
company_keys: Sequence[str] = (), places: Sequence[str] = (),
seniority: Sequence[str] = (), seniority_unknown: bool = True,
pay_unknown: bool = False, by_cv: bool = False, closing: bool = False,
@@ -1264,6 +1279,7 @@ class Store:
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
fetch_gate=fetch_gate,
judged=judged, pay=pay, company_keys=company_keys, places=places,
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
closing=closing)
@@ -1280,7 +1296,9 @@ class Store:
if scored:
select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt, s.cv_fit"
order = (_by_cv(cv) if by_cv else "") + "score DESC, " + order
elif open_only:
elif agent is not None and fetch_gate:
select += ", s.gated_reason"
if not scored and open_only:
# Same reasoning as `roles`: the planner otherwise sorts every
# open role before applying LIMIT.
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
@@ -1318,7 +1336,8 @@ class Store:
days: int | None = None, hide_muted: bool = False,
include_read: bool = True, tags: Sequence[str] = (),
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
gated: bool = False, judged: str | None = None,
gated: bool = False, fetch_gate: bool = False,
judged: str | None = None,
pay: str | None = None, company_keys: Sequence[str] = (),
places: Sequence[str] = (), seniority: Sequence[str] = (),
seniority_unknown: bool = True, pay_unknown: bool = False,
@@ -1328,6 +1347,7 @@ class Store:
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
fetch_gate=fetch_gate,
judged=judged, pay=pay, company_keys=company_keys, places=places,
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
closing=closing)
@@ -1340,7 +1360,8 @@ class Store:
days: int | None = None, hide_muted: bool = False,
include_read: bool = True, tags: Sequence[str] = (),
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
gated: bool = False, judged: str | None = None,
gated: bool = False, fetch_gate: bool = False,
judged: str | None = None,
pay: str | None = None, company_keys: Sequence[str] = (),
places: Sequence[str] = (), seniority: Sequence[str] = (),
seniority_unknown: bool = True,
@@ -1356,7 +1377,7 @@ class Store:
"company": company, "place": place, "open_only": open_only, "q": q,
"function": function, "remote": remote, "days": days, "hide_muted": hide_muted,
"include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent,
"by_score": by_score, "gated": gated, "judged": judged,
"by_score": by_score, "gated": gated, "fetch_gate": fetch_gate, "judged": judged,
"company_keys": company_keys, "places": places, "closing": closing}
by_level, level_params = self._roles_from(
**shared, pay=pay, pay_unknown=pay_unknown)
@@ -1376,6 +1397,7 @@ class Store:
days: int | None, hide_muted: bool, include_read: bool,
tags: Sequence[str], tag_mode: str, agent: int | None,
by_score: bool, gated: bool, judged: str | None,
fetch_gate: bool = False,
pay: str | None = None, company_keys: Sequence[str] = (),
places: Sequence[str] = (), seniority: Sequence[str] = (),
seniority_unknown: bool = True,
@@ -1384,7 +1406,9 @@ class Store:
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
its parameters in order. The agent's votes are always joined as `v`
(see `_WITH_VOTES`) and its judgements as `j` (`_WITH_JUDGEMENTS`);
its scores as `s` only when ordering by them. A copy of another role
its scores as `s` when ordering by them or with `fetch_gate`, and
then a role the agent's fetch gate rejects is left out unless
`gated`. A copy of another role
(`rebuild_dupes`) is never listed: its kept role stands for it and
counts its copies. `closing` keeps the roles whose deadline is
today or within `CLOSING_DAYS` after it."""
@@ -1458,7 +1482,7 @@ class Store:
params += values
sql = f"FROM roles {_WITH_VOTES} {_WITH_JUDGEMENTS}"
joins: list[Any] = [agent, agent]
if agent is not None and by_score:
if agent is not None and (by_score or fetch_gate):
sql += " JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?"
joins.append(agent)
if not gated:
@@ -1639,21 +1663,18 @@ class Store:
# -------------------------------------------------------------- extract
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
"""The `WHERE` fragment for a role among the best ungated scores of
an enabled agent, as deep and as low as that agent's own `limits`
say for `stage`: what the two model stages, extract and summarise,
are bounded to. The rank is numbered per agent and the agent's
limits ride out of the window subquery beside it, so each row is
held to the limits of the agent that ranked it; a key the JSON lacks
is the `Limits` default. The role is `r`. A company an agent hid
needs no clause: `score` writes no rows for it under that agent. An
agent without a prompt with text counts for nothing here: its scores
have no fit, so they say how fresh a role is, not whether it is
wanted."""
defaults = Limits()
return (f"""r.uid IN (SELECT uid FROM (
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
def _summarise_top(self) -> tuple[str, list[Any]]:
"""The `WHERE` fragment for a role among the best `summarise_top`
final scores of an enabled agent with a prompt, open, no copy and
passing that agent's fetch gate: the top of its feed. The rank is
numbered per agent and the agent's limits ride out of the window
subquery beside it, so each row is held to the limit of the agent
that ranked it; a key the JSON lacks is the `Limits` default. The
role is `r`. An agent without a prompt with text counts for nothing
here: its scores have no fit, so they say how fresh a role is, not
whether it is wanted."""
return ("""r.uid IN (SELECT uid FROM (
SELECT s.uid, a.limits, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s
@@ -1664,15 +1685,14 @@ class Store:
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
AND top.dup_of IS NULL
WHERE s.gated_reason = '')
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
[getattr(defaults, f"{stage}_top"), getattr(defaults, f"{stage}_min")])
WHERE n <= COALESCE(json_extract(limits, '$.summarise_top'), ?))""",
[Limits().summarise_top])
def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
"""The `WHERE` for a role `extract` may read: open, never answered,
not given up. Given `uids`, those of them and nothing else, since a
batch asks again when it runs and must finish what it was dealt;
otherwise only the roles in `_top_scope`."""
otherwise only the roles in the cut of an enabled agent (`_IN_CUT`)."""
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
.replace("dup_of", "r.dup_of")
@@ -1682,8 +1702,7 @@ class Store:
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
top, params = self._top_scope("extract")
return " AND ".join([*where, top]), params
return " AND ".join([*where, _IN_CUT]), []
def extract_pending(self) -> int:
where, params = self._extract_scope(None)
@@ -1691,14 +1710,13 @@ class Store:
return int(row["n"]) if row else 0
def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `extract` has yet to read: the best of every enabled agent's
scores, as deep and as low as that agent's own `limits` say, so the
backlog follows the rankings rather than the boards. Ordered by the
best rank any agent gives the role (each agent's first, then each
agent's second ...), then by its score, so every agent's top comes
before anyone's tail whatever limits cut the batch. Given `uids`,
only those of them still waiting, whatever their rank; a role no agent
ranks comes last."""
"""Roles `extract` has yet to read: those in the cut of an enabled
agent, so the backlog follows the cut rather than the boards.
Ordered by the best rank any agent gives the role by final score
(each agent's first, then each agent's second ...), then by that
score, so every agent's top comes before anyone's tail whatever
limit cuts the batch. Given `uids`, only those of them still waiting,
whatever their rank; a role no agent ranks comes last."""
where, params = self._extract_scope(uids)
return [Role.from_row(r) for r in self._rows(
"SELECT r.* FROM roles r LEFT JOIN ("
@@ -1826,15 +1844,16 @@ class Store:
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
some text to summarise. Given `uids`, those of them and nothing else,
open or not and given up or not, because somebody asked for them.
Otherwise open, not given up, and in `_top_scope` or one the owner
cares about whatever its rank: saved (it has an application) or voted
up by an enabled agent."""
Otherwise open, not given up, and in the top of an enabled agent's
feed (`_summarise_top`) or one the owner cares about whatever its
rank: saved (it has an application) or voted up by an enabled
agent."""
where = ["r.summary = ''",
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
top, params = self._top_scope("summarise")
top, params = self._summarise_top()
where += [
"r.closed_at IS NULL",
"r.dup_of IS NULL",
@@ -1856,12 +1875,12 @@ class Store:
return int(row["n"]) if row else 0
def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `summarise` has yet to write, newest first: the top of every
enabled agent's scores, as deep and as low as that agent's own
`limits` say, so the backlog is bounded by the agents rather than by
the number of open roles, and every open role that is saved or voted
up by an enabled agent, whatever its rank; given `uids`, those still without a
summary, whatever their rank. A batch asks again when it runs."""
"""Roles `summarise` has yet to write, newest first: the best
`summarise_top` final scores of every enabled agent, so the backlog
is bounded by the agents rather than by the number of open roles,
and every open role that is saved or voted up by an enabled agent,
whatever its rank; given `uids`, those still without a summary,
whatever their rank. A batch asks again when it runs."""
where, params = self._summary_scope(uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
@@ -1908,17 +1927,15 @@ class Store:
# ---------------------------------------------------------------- judge
def judge_rows(self, agent: Agent, uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
"""What `judge` may read for one agent: its best ungated scores of
open roles that are no copy, as deep and as low as its `limits` say
(`judge_top`, `judge_min`), best first, then the `judge_cv_top`
ungated ones closest to its CV (`scores.cv_fit`) that the first set
leaves out, closest first, each with some text (`article_text` or
`description`). Given `uids`, those of them with
text, whatever their rank or state. Each row carries `uid`,
`title`, `company`, `place`, `article_text`, `description`, and the
stored judgement's `input_key`, `judged_at` and `tries` (NULL, NULL
and 0 without one), and `given_up` once `tries` reached
`GIVE_UP_AFTER`."""
"""What `judge` may read for one agent: the open roles in its cut
that are no copy and have some text (`article_text` or
`description`), those in by fit first, best fit first, then those in
by CV closeness, closest first, then those kept because they are
liked or saved. Given `uids`, those of them with text, whatever
their rank or state. Each row carries `uid`, `title`, `company`,
`place`, `article_text`, `description`, and the stored judgement's
`input_key`, `judged_at` and `tries` (NULL, NULL and 0 without one),
and `given_up` once `tries` reached `GIVE_UP_AFTER`."""
text = "(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"
select = ("SELECT r.uid, r.title, r.company, r.place, r.article_text, r.description, "
"j.input_key, j.judged_at, COALESCE(j.tries, 0) AS tries, "
@@ -1928,23 +1945,13 @@ class Store:
return self._rows(
f"{select} roles r {joined} WHERE r.uid IN (SELECT value FROM json_each(?)) "
f"AND {text} ORDER BY r.uid", (agent.id, json.dumps(list(uids))))
ranked = ("FROM scores s JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL "
"AND top.dup_of IS NULL WHERE s.agent_id = ? AND s.gated_reason = ''")
return self._rows(
f"""WITH best AS (SELECT s.uid, s.score {ranked}
ORDER BY s.score DESC, top.posted_at DESC, s.uid LIMIT ?),
by_score AS (SELECT uid, 0 AS part, score AS rank FROM best
WHERE score >= ?),
by_cv AS (SELECT s.uid, 1 AS part, s.cv_fit AS rank {ranked}
AND s.cv_fit IS NOT NULL
AND s.uid NOT IN (SELECT uid FROM by_score)
ORDER BY s.cv_fit DESC, top.posted_at DESC, s.uid LIMIT ?)
{select} (SELECT * FROM by_score UNION ALL SELECT * FROM by_cv) w
JOIN roles r ON r.uid = w.uid {joined}
WHERE {text}
ORDER BY w.part, w.rank DESC, r.posted_at DESC, r.uid""",
(agent.id, agent.limits.judge_top, agent.limits.judge_min, agent.id,
agent.limits.judge_cv_top, agent.id))
f"""{select} scores s JOIN roles r ON r.uid = s.uid AND r.closed_at IS NULL
AND r.dup_of IS NULL
{joined}
WHERE s.agent_id = ? AND s.cut <> '' AND {text}
ORDER BY {_CUT_ORDER}, r.posted_at DESC, r.uid""",
(agent.id, agent.id))
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
experience: int, career: int, strengths: Sequence[str],
@@ -2031,18 +2038,15 @@ class Store:
keeps it."""
return bool(self._write("UPDATE roles SET posting_url = ? WHERE uid = ?", (url, uid)))
# --------------------------------------------------------------- enrich
# ----------------------------------------------------------- fetch text
def enrich_waiting(self, top: int, floor: float,
uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
"""Open board and search roles without text that an enabled agent with a prompt
ranks ungated among its best `top` and at `floor` or above, best
first by the highest such score, as `{uid, source_id,
subscription_id, title, url, published_at, raw, score}`. The ranking
is `_top_scope`'s, with the bar given rather than the agent's own.
Given `uids`, those of them still without text, whatever their rank,
in no order (`score` is None), since a batch finishes what it was
dealt."""
def fetch_text_waiting(self, uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
"""Open board and search roles without text in the cut of an enabled
agent, best first by the highest final score any such agent gives
them, as `{uid, source_id, subscription_id, title, url, published_at,
raw, score}`. Given `uids`, those of them still without text, cut or
not, in no order (`score` is None), since a batch finishes what it
was dealt."""
fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, "
"r.published_at, r.raw")
waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source IN ('ats', 'search') "
@@ -2054,21 +2058,11 @@ class Store:
"AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),))
else:
rows = self._rows(
f"""SELECT {fields}, MAX(ranked.score) AS score FROM (
SELECT s.uid, s.score, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
AND EXISTS (SELECT 1 FROM agent_prompts p
WHERE p.agent_id = a.id
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
AND top.dup_of IS NULL
WHERE s.gated_reason = '') ranked
JOIN roles r ON r.uid = ranked.uid
WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting}
GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor))
f"""SELECT {fields}, MAX(s.score) AS score FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles r ON r.uid = s.uid
WHERE s.cut <> '' AND {waiting}
GROUP BY r.uid ORDER BY score DESC, r.uid""")
return [dict(row) for row in rows]
# -------------------------------------------------------------- vectors
@@ -2220,10 +2214,41 @@ class Store:
return found
def delete_agent(self, agent_id: int) -> bool:
"""Its prompts, scores, votes and affinities go with it, by the
foreign keys."""
"""Its prompts, scores, votes, affinities and saved filters go with
it, by the foreign keys."""
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
# -------------------------------------------------------- saved filters
def saved_filters(self, agent_id: int) -> list[SavedFilter]:
"""The agent's saved filters, by name regardless of case."""
return [SavedFilter.from_row(r) for r in self._rows(
"SELECT * FROM saved_filters WHERE agent_id=? ORDER BY name COLLATE NOCASE, id",
(agent_id,))]
def save_filter(self, agent_id: int, name: str, query: str) -> SavedFilter:
"""Save the feed's filters under `name` for the agent. ValueError on
an empty name or one the agent already has regardless of case,
KeyError on an unknown agent."""
name = " ".join(name.split())
if not name:
raise ValueError("a saved filter needs a name")
try:
with self.transaction() as conn:
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
raise KeyError(agent_id)
new_id = conn.execute(
"INSERT INTO saved_filters (agent_id, name, query, created_at) "
"VALUES (?,?,?,?)", (agent_id, name, query, self.now())).lastrowid
except sqlite3.IntegrityError as exc:
raise ValueError(f"a filter named {name!r} is already saved") from exc
row = self._row("SELECT * FROM saved_filters WHERE id=?", (new_id,))
assert row is not None
return SavedFilter.from_row(row)
def delete_filter(self, filter_id: int) -> bool:
return self._write("DELETE FROM saved_filters WHERE id=?", (filter_id,)) > 0
def prompt_vector(self, agent_id: int, name: str, key: str) -> np.ndarray | None:
"""The cached vector of the agent's prompt `name`, if it was made
under `key`."""
@@ -2262,11 +2287,11 @@ class Store:
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
"""Every open role of a company this agent has not hidden, copies
(`rebuild_dupes`) aside, with the agent's affinity for the company
and how the role meets each gate: 1 match, 0 no match, -1 the column
is empty. The gates are evaluated here, in SQL, so a `require` gate is
the `WHERE` it reads as, written
as a column so a gated role keeps its row and reason."""
(`rebuild_dupes`) aside, with the agent's affinity for the company,
the company's All countries and All roles, and how the role meets
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
as, written as a column so a gated role keeps its row and reason."""
columns: list[str] = []
params: list[Any] = []
gaz = self.gazetteer()
@@ -2281,31 +2306,41 @@ class Store:
params += values
columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}")
select = ", ".join(["r.uid", "r.posted_at",
"COALESCE(ac.affinity, 0) AS affinity", *columns])
"COALESCE(ac.affinity, 0) AS affinity",
"COALESCE(c.all_countries, 0) AS all_countries",
"COALESCE(c.all_roles, 0) AS all_roles", *columns])
rows = self._rows(
f"""SELECT {select} FROM roles r
LEFT JOIN agent_companies ac
ON ac.company_key = r.company_key AND ac.agent_id = ?
LEFT JOIN companies c ON c.key = r.company_key
WHERE r.closed_at IS NULL AND r.dup_of IS NULL
AND COALESCE(ac.affinity, 0) <> -1""",
[*params, agent_id])
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
affinity=int(r["affinity"]),
all_countries=bool(r["all_countries"]), all_roles=bool(r["all_roles"]),
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
for r in rows]
def saved_uids(self) -> set[str]:
"""The roles saved (an application names them), which make every
agent's cut whatever their levels."""
return {r["role_uid"] for r in self._rows(
"SELECT DISTINCT role_uid FROM applications WHERE role_uid IS NOT NULL")}
def replace_scores(self, agent_id: int,
rows: Sequence[tuple[str, float, str] | tuple[str, float, str, str]
| tuple[str, float, str, str, float | None]]
) -> int:
rows: Sequence[tuple[Any, ...]]) -> int:
"""`(uid, score, gated_reason)` for every role this agent scored,
with the name of the prompt that gave the fit as a fourth item ('' when
left out) and the role's closeness to the agent's CV as a fifth (None
when left out), replacing whatever it had: a role closed or muted since the last run
drops out rather than keeping a stale score. One transaction, so a
reader sees the old list or the new one. An agent deleted while its
run was scoring gets nothing written and 0 back: its rows are gone
with it, and an insert would only fail the foreign key."""
and optionally, in this order, the name of the prompt that gave the
fit ('' when left out), the role's closeness to the agent's CV, its
prompt fit level (both None when left out) and why it is in the cut
('' when left out: not in it). Replaces whatever the agent had: a
role closed or muted since the last run drops out rather than keeping
a stale score. One transaction, so a reader sees the old list or the
new one. An agent deleted while its run was scoring gets nothing
written and 0 back: its rows are gone with it, and an insert would
only fail the foreign key."""
stamp = self.now()
with self.transaction() as conn:
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
@@ -2313,13 +2348,20 @@ class Store:
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
conn.executemany(
"""INSERT INTO scores (uid, agent_id, score, gated_reason, prompt, cv_fit,
scored_at)
VALUES (?,?,?,?,?,?,?)""",
fit, cut, scored_at)
VALUES (?,?,?,?,?,?,?,?,?)""",
[(row[0], agent_id, row[1], row[2], row[3] if len(row) > 3 else "",
row[4] if len(row) > 4 else None, stamp)
row[4] if len(row) > 4 else None, row[5] if len(row) > 5 else None,
row[6] if len(row) > 6 else "", stamp)
for row in rows])
return len(rows)
def cut(self, agent_id: int) -> dict[str, str]:
"""`uid -> why` for the roles in one agent's cut: 'fit', 'cv' or
'kept'."""
return {r["uid"]: str(r["cut"]) for r in self._rows(
"SELECT uid, cut FROM scores WHERE agent_id=? AND cut <> ''", (agent_id,))}
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
"""`uid -> (score, gated_reason)` for one agent."""
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))