Rank by the cut: fetch gate overrides in scoring, the cut per agent, stages read the cut, fetch text renamed from enrich, the feed kept to the fetch gate, and migration 32
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
060ccf66e6
commit
617356e60c
35 files changed
+1054
-751
No files matched your search
+101
-4
@@ -385,10 +385,14 @@ CREATE TABLE IF NOT EXISTS role_votes (
|
||||
PRIMARY KEY (agent_id, uid)
|
||||
);
|
||||
|
||||
-- A role is scored per agent. A gated role keeps its row, with the reason.
|
||||
-- prompt names the agent's prompt that gave the fit, '' when fit did not count.
|
||||
-- cv_fit is how close the role is to the agent's CV, 0..1, NULL without a CV
|
||||
-- or a vector and for a gated role; it is no term of score.
|
||||
-- A role is scored per agent. A role the fetch gate rejects keeps its row,
|
||||
-- with the reason. score is the final score, the feed's order. prompt names
|
||||
-- the agent's prompt that gave the fit, '' when fit did not count. fit is the
|
||||
-- prompt fit level and cv_fit how close the role is to the agent's CV, both
|
||||
-- 0..1 before the affinity bonus, NULL without a prompt, a CV or a vector and
|
||||
-- for a gated role. cut says why the role is in the agent's cut, the roles
|
||||
-- that get fetch text, extract and judge: 'fit', 'cv', 'kept' (liked or
|
||||
-- saved), or '' when it is not.
|
||||
CREATE TABLE IF NOT EXISTS scores (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||||
@@ -397,9 +401,23 @@ CREATE TABLE IF NOT EXISTS scores (
|
||||
scored_at TEXT NOT NULL,
|
||||
prompt TEXT NOT NULL DEFAULT '',
|
||||
cv_fit REAL,
|
||||
fit REAL,
|
||||
cut TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (uid, agent_id)
|
||||
);
|
||||
|
||||
-- The feed's filter settings saved under a name, per agent. query is the
|
||||
-- feed's query string without the agent. Names are unique per agent
|
||||
-- regardless of case; a change is a new row.
|
||||
CREATE TABLE IF NOT EXISTS saved_filters (
|
||||
id INTEGER PRIMARY KEY,
|
||||
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
query TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE (agent_id, name COLLATE NOCASE)
|
||||
);
|
||||
|
||||
-- The vectors of an agent's CV, one per chunk in order, for scores.cv_fit.
|
||||
-- vec_key is sha1 of the embedding model, the chunking and the CV text, the
|
||||
-- same on every row of an agent, so an unchanged CV is not embedded again.
|
||||
@@ -487,6 +505,9 @@ CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id
|
||||
CREATE INDEX IF NOT EXISTS idx_application_events
|
||||
ON application_events(application_id, at);
|
||||
CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
|
||||
-- The roles in an agent's cut, which fetch text, extract and judge read.
|
||||
-- Partial: most roles are in no cut.
|
||||
CREATE INDEX IF NOT EXISTS idx_scores_cut ON scores(agent_id, uid) WHERE cut <> '';
|
||||
CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote);
|
||||
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
|
||||
@@ -731,6 +752,12 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
# missing. The next score fills them.
|
||||
(31, "scores.cv_fit and agent_cv_vectors: how close each role is to the agent's CV",
|
||||
lambda conn: _add_cv_fit(conn)),
|
||||
# A Python step: gates and limits are JSON. The old limits were bars on
|
||||
# the blended score and mean nothing on plain fit, so every agent takes
|
||||
# the new defaults.
|
||||
(32, "the cut: scores.fit and scores.cut, agents' limits reset to the cut's defaults, "
|
||||
"prefer gates gone, saved_filters, and enrich renamed fetch text",
|
||||
lambda conn: _add_cut(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -1094,6 +1121,76 @@ def _add_cv_fit(conn: sqlite3.Connection) -> None:
|
||||
PRIMARY KEY (agent_id, position))""")
|
||||
|
||||
|
||||
#: What migration 32 writes into every agent's `limits`: `models.Limits` as
|
||||
#: it stood then, spelled out so a later change of the defaults does not
|
||||
#: change what this migration did.
|
||||
_V32_LIMITS = {"cut_fit_top": 500, "cut_fit_min": 0.6, "cut_cv_top": 150, "cut_cv_min": 0.6,
|
||||
"summarise_top": 50}
|
||||
|
||||
_V32 = """
|
||||
CREATE TABLE IF NOT EXISTS saved_filters (
|
||||
id INTEGER PRIMARY KEY,
|
||||
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
query TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE (agent_id, name COLLATE NOCASE)
|
||||
);
|
||||
INSERT OR IGNORE INTO state (key, value)
|
||||
SELECT 'stage:fetch_text', value FROM state WHERE key = 'stage:enrich';
|
||||
DELETE FROM state WHERE key = 'stage:enrich';
|
||||
"""
|
||||
|
||||
|
||||
def _add_cut(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 32's step. `scores.fit` and `scores.cut`, NULL and empty
|
||||
until the next score; every agent's `limits` reset to `_V32_LIMITS`;
|
||||
the `prefer` gates dropped from every agent's `gates` and `weight` from
|
||||
the others; `saved_filters`; the switch `stage:enrich` renamed
|
||||
`stage:fetch_text`; and the queued `enrich` and `enrich_batch` jobs
|
||||
deleted, with a schedule of `enrich` firing `fetch_text`. Runs twice
|
||||
without harm: limits are reset when the columns are added, and after
|
||||
that only where they still hold a key the cut does not know."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(scores)")}
|
||||
fresh = "cut" not in columns
|
||||
if "fit" not in columns:
|
||||
conn.execute("ALTER TABLE scores ADD COLUMN fit REAL")
|
||||
if fresh:
|
||||
conn.execute("ALTER TABLE scores ADD COLUMN cut TEXT NOT NULL DEFAULT ''")
|
||||
reset = dropped = 0
|
||||
for agent_id, raw_limits, raw_gates in conn.execute(
|
||||
"SELECT id, limits, gates FROM agents").fetchall():
|
||||
try:
|
||||
limits = json.loads(raw_limits or "{}")
|
||||
except ValueError:
|
||||
limits = None
|
||||
if fresh or not isinstance(limits, dict) or set(limits) - set(_V32_LIMITS):
|
||||
conn.execute("UPDATE agents SET limits=? WHERE id=?",
|
||||
(json.dumps(_V32_LIMITS), agent_id))
|
||||
reset += 1
|
||||
try:
|
||||
gates = json.loads(raw_gates or "[]")
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(gates, list):
|
||||
continue
|
||||
kept = [{k: v for k, v in g.items() if k != "weight"} for g in gates
|
||||
if isinstance(g, dict) and g.get("mode") != "prefer"]
|
||||
if kept != gates:
|
||||
dropped += sum(1 for g in gates if isinstance(g, dict) and g.get("mode") == "prefer")
|
||||
conn.execute("UPDATE agents SET gates=? WHERE id=?", (json.dumps(kept), agent_id))
|
||||
for statement in _statements(_V32):
|
||||
conn.execute(statement)
|
||||
tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
if "jobs" in tables:
|
||||
conn.execute("DELETE FROM jobs WHERE state = 'queued' "
|
||||
"AND kind IN ('enrich', 'enrich_batch')")
|
||||
if "schedules" in tables:
|
||||
conn.execute("UPDATE schedules SET kind = 'fetch_text' WHERE kind = 'enrich'")
|
||||
log.info("migration 32: %d agents' limits reset to the cut's defaults, %d prefer gates "
|
||||
"dropped", reset, dropped)
|
||||
|
||||
|
||||
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 21's step. The column; the tag for every company that is a
|
||||
favourite of any agent, since favourite no longer widens what a pull
|
||||
|
||||
+167
-125
@@ -25,7 +25,7 @@ from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
|
||||
RoleCopy, RolePlace, RoleRow, ScoredRole, cv_languages as normal_languages, cv_score,
|
||||
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
|
||||
BROKEN_CAP, capped_career, language_flag, role_languages,
|
||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
|
||||
@@ -42,7 +42,7 @@ _RAW_TEXT = ("place", "currency")
|
||||
#: The host's own words a reader hands on in `Item.raw`, stored as
|
||||
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
|
||||
#: rewritten every pull, as the host's level always was; the others are kept
|
||||
#: filled, since a role's own page (`enrich`) may state what its listing
|
||||
#: filled, since a role's own page (`fetch_text`) may state what its listing
|
||||
#: does not.
|
||||
_HOST_WORDS = ("seniority", "employment", "remote", "country")
|
||||
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
|
||||
@@ -78,6 +78,16 @@ _EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_whe
|
||||
"OR languages_required IS NULL) "
|
||||
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
#: A role in the cut of an enabled agent (`scores.cut`, see `app.jobs.score`),
|
||||
#: as a `WHERE` on `r`: what fetch text and extract read.
|
||||
_IN_CUT = ("r.uid IN (SELECT s.uid FROM scores s JOIN agents a ON a.id = s.agent_id "
|
||||
"AND a.enabled = 1 WHERE s.cut <> '')")
|
||||
|
||||
#: The order within one agent's cut, over its scores as `s`: in by fit, best
|
||||
#: fit first, then in by CV closeness, closest first, then kept.
|
||||
_CUT_ORDER = ("CASE s.cut WHEN 'fit' THEN 0 WHEN 'cv' THEN 1 ELSE 2 END, "
|
||||
"CASE s.cut WHEN 'fit' THEN s.fit WHEN 'cv' THEN s.cv_fit END DESC")
|
||||
|
||||
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
||||
VOTES = (-1, 0, 1, None)
|
||||
|
||||
@@ -1127,7 +1137,7 @@ class Store:
|
||||
hide_muted: bool = False, include_read: bool = True,
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, pay: str | None = None,
|
||||
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
|
||||
company_keys: Sequence[str] = (), places: Sequence[str] = (),
|
||||
seniority: Sequence[str] = (), seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False, by_cv: bool = False, closing: bool = False,
|
||||
@@ -1172,7 +1182,9 @@ class Store:
|
||||
set. The score is the stored one moved by the agent's vote
|
||||
(`_VOTED_SCORE`), so a vote shows in the order at once and needs no
|
||||
rescore. With `by_score` off the order is newest, with the votes and
|
||||
no score.
|
||||
no score; with `fetch_gate` as well, the newest order keeps to the
|
||||
agent's fetch gate the same way, carrying `gated_reason`, so no order
|
||||
shows a role the agent's gate rejects unless `gated`.
|
||||
|
||||
With `by_cv` as well, the roles the agent's `judge` read come first
|
||||
by `cv_score`, then the others by how close they are to the CV
|
||||
@@ -1191,6 +1203,7 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
fetch_gate=fetch_gate,
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
||||
closing=closing)
|
||||
@@ -1202,7 +1215,9 @@ class Store:
|
||||
if scored:
|
||||
select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt, s.cv_fit"
|
||||
order = (_by_cv(cv) if by_cv else "") + "score DESC, " + order
|
||||
elif open_only:
|
||||
elif agent is not None and fetch_gate:
|
||||
select += ", s.gated_reason"
|
||||
if not scored and open_only:
|
||||
# Newest first, open only: idx_roles_posted covers this exactly
|
||||
# (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
|
||||
# LIMIT rather than sorting every open role first. Left to the
|
||||
@@ -1241,7 +1256,7 @@ class Store:
|
||||
hide_muted: bool = False, include_read: bool = True,
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, pay: str | None = None,
|
||||
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
|
||||
company_keys: Sequence[str] = (), places: Sequence[str] = (),
|
||||
seniority: Sequence[str] = (), seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False, by_cv: bool = False, closing: bool = False,
|
||||
@@ -1264,6 +1279,7 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
fetch_gate=fetch_gate,
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
||||
closing=closing)
|
||||
@@ -1280,7 +1296,9 @@ class Store:
|
||||
if scored:
|
||||
select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt, s.cv_fit"
|
||||
order = (_by_cv(cv) if by_cv else "") + "score DESC, " + order
|
||||
elif open_only:
|
||||
elif agent is not None and fetch_gate:
|
||||
select += ", s.gated_reason"
|
||||
if not scored and open_only:
|
||||
# Same reasoning as `roles`: the planner otherwise sorts every
|
||||
# open role before applying LIMIT.
|
||||
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
|
||||
@@ -1318,7 +1336,8 @@ class Store:
|
||||
days: int | None = None, hide_muted: bool = False,
|
||||
include_read: bool = True, tags: Sequence[str] = (),
|
||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||
gated: bool = False, judged: str | None = None,
|
||||
gated: bool = False, fetch_gate: bool = False,
|
||||
judged: str | None = None,
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True, pay_unknown: bool = False,
|
||||
@@ -1328,6 +1347,7 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
fetch_gate=fetch_gate,
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
||||
closing=closing)
|
||||
@@ -1340,7 +1360,8 @@ class Store:
|
||||
days: int | None = None, hide_muted: bool = False,
|
||||
include_read: bool = True, tags: Sequence[str] = (),
|
||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||
gated: bool = False, judged: str | None = None,
|
||||
gated: bool = False, fetch_gate: bool = False,
|
||||
judged: str | None = None,
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True,
|
||||
@@ -1356,7 +1377,7 @@ class Store:
|
||||
"company": company, "place": place, "open_only": open_only, "q": q,
|
||||
"function": function, "remote": remote, "days": days, "hide_muted": hide_muted,
|
||||
"include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent,
|
||||
"by_score": by_score, "gated": gated, "judged": judged,
|
||||
"by_score": by_score, "gated": gated, "fetch_gate": fetch_gate, "judged": judged,
|
||||
"company_keys": company_keys, "places": places, "closing": closing}
|
||||
by_level, level_params = self._roles_from(
|
||||
**shared, pay=pay, pay_unknown=pay_unknown)
|
||||
@@ -1376,6 +1397,7 @@ class Store:
|
||||
days: int | None, hide_muted: bool, include_read: bool,
|
||||
tags: Sequence[str], tag_mode: str, agent: int | None,
|
||||
by_score: bool, gated: bool, judged: str | None,
|
||||
fetch_gate: bool = False,
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True,
|
||||
@@ -1384,7 +1406,9 @@ class Store:
|
||||
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
||||
its parameters in order. The agent's votes are always joined as `v`
|
||||
(see `_WITH_VOTES`) and its judgements as `j` (`_WITH_JUDGEMENTS`);
|
||||
its scores as `s` only when ordering by them. A copy of another role
|
||||
its scores as `s` when ordering by them or with `fetch_gate`, and
|
||||
then a role the agent's fetch gate rejects is left out unless
|
||||
`gated`. A copy of another role
|
||||
(`rebuild_dupes`) is never listed: its kept role stands for it and
|
||||
counts its copies. `closing` keeps the roles whose deadline is
|
||||
today or within `CLOSING_DAYS` after it."""
|
||||
@@ -1458,7 +1482,7 @@ class Store:
|
||||
params += values
|
||||
sql = f"FROM roles {_WITH_VOTES} {_WITH_JUDGEMENTS}"
|
||||
joins: list[Any] = [agent, agent]
|
||||
if agent is not None and by_score:
|
||||
if agent is not None and (by_score or fetch_gate):
|
||||
sql += " JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?"
|
||||
joins.append(agent)
|
||||
if not gated:
|
||||
@@ -1639,21 +1663,18 @@ class Store:
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` fragment for a role among the best ungated scores of
|
||||
an enabled agent, as deep and as low as that agent's own `limits`
|
||||
say for `stage`: what the two model stages, extract and summarise,
|
||||
are bounded to. The rank is numbered per agent and the agent's
|
||||
limits ride out of the window subquery beside it, so each row is
|
||||
held to the limits of the agent that ranked it; a key the JSON lacks
|
||||
is the `Limits` default. The role is `r`. A company an agent hid
|
||||
needs no clause: `score` writes no rows for it under that agent. An
|
||||
agent without a prompt with text counts for nothing here: its scores
|
||||
have no fit, so they say how fresh a role is, not whether it is
|
||||
wanted."""
|
||||
defaults = Limits()
|
||||
return (f"""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
|
||||
def _summarise_top(self) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` fragment for a role among the best `summarise_top`
|
||||
final scores of an enabled agent with a prompt, open, no copy and
|
||||
passing that agent's fetch gate: the top of its feed. The rank is
|
||||
numbered per agent and the agent's limits ride out of the window
|
||||
subquery beside it, so each row is held to the limit of the agent
|
||||
that ranked it; a key the JSON lacks is the `Limits` default. The
|
||||
role is `r`. An agent without a prompt with text counts for nothing
|
||||
here: its scores have no fit, so they say how fresh a role is, not
|
||||
whether it is wanted."""
|
||||
return ("""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, a.limits, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.agent_id
|
||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
@@ -1664,15 +1685,14 @@ class Store:
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
AND top.dup_of IS NULL
|
||||
WHERE s.gated_reason = '')
|
||||
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
|
||||
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
|
||||
[getattr(defaults, f"{stage}_top"), getattr(defaults, f"{stage}_min")])
|
||||
WHERE n <= COALESCE(json_extract(limits, '$.summarise_top'), ?))""",
|
||||
[Limits().summarise_top])
|
||||
|
||||
def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` for a role `extract` may read: open, never answered,
|
||||
not given up. Given `uids`, those of them and nothing else, since a
|
||||
batch asks again when it runs and must finish what it was dealt;
|
||||
otherwise only the roles in `_top_scope`."""
|
||||
otherwise only the roles in the cut of an enabled agent (`_IN_CUT`)."""
|
||||
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
|
||||
.replace("dup_of", "r.dup_of")
|
||||
@@ -1682,8 +1702,7 @@ class Store:
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
top, params = self._top_scope("extract")
|
||||
return " AND ".join([*where, top]), params
|
||||
return " AND ".join([*where, _IN_CUT]), []
|
||||
|
||||
def extract_pending(self) -> int:
|
||||
where, params = self._extract_scope(None)
|
||||
@@ -1691,14 +1710,13 @@ class Store:
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `extract` has yet to read: the best of every enabled agent's
|
||||
scores, as deep and as low as that agent's own `limits` say, so the
|
||||
backlog follows the rankings rather than the boards. Ordered by the
|
||||
best rank any agent gives the role (each agent's first, then each
|
||||
agent's second ...), then by its score, so every agent's top comes
|
||||
before anyone's tail whatever limits cut the batch. Given `uids`,
|
||||
only those of them still waiting, whatever their rank; a role no agent
|
||||
ranks comes last."""
|
||||
"""Roles `extract` has yet to read: those in the cut of an enabled
|
||||
agent, so the backlog follows the cut rather than the boards.
|
||||
Ordered by the best rank any agent gives the role by final score
|
||||
(each agent's first, then each agent's second ...), then by that
|
||||
score, so every agent's top comes before anyone's tail whatever
|
||||
limit cuts the batch. Given `uids`, only those of them still waiting,
|
||||
whatever their rank; a role no agent ranks comes last."""
|
||||
where, params = self._extract_scope(uids)
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
"SELECT r.* FROM roles r LEFT JOIN ("
|
||||
@@ -1826,15 +1844,16 @@ class Store:
|
||||
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
|
||||
some text to summarise. Given `uids`, those of them and nothing else,
|
||||
open or not and given up or not, because somebody asked for them.
|
||||
Otherwise open, not given up, and in `_top_scope` or one the owner
|
||||
cares about whatever its rank: saved (it has an application) or voted
|
||||
up by an enabled agent."""
|
||||
Otherwise open, not given up, and in the top of an enabled agent's
|
||||
feed (`_summarise_top`) or one the owner cares about whatever its
|
||||
rank: saved (it has an application) or voted up by an enabled
|
||||
agent."""
|
||||
where = ["r.summary = ''",
|
||||
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
top, params = self._top_scope("summarise")
|
||||
top, params = self._summarise_top()
|
||||
where += [
|
||||
"r.closed_at IS NULL",
|
||||
"r.dup_of IS NULL",
|
||||
@@ -1856,12 +1875,12 @@ class Store:
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `summarise` has yet to write, newest first: the top of every
|
||||
enabled agent's scores, as deep and as low as that agent's own
|
||||
`limits` say, so the backlog is bounded by the agents rather than by
|
||||
the number of open roles, and every open role that is saved or voted
|
||||
up by an enabled agent, whatever its rank; given `uids`, those still without a
|
||||
summary, whatever their rank. A batch asks again when it runs."""
|
||||
"""Roles `summarise` has yet to write, newest first: the best
|
||||
`summarise_top` final scores of every enabled agent, so the backlog
|
||||
is bounded by the agents rather than by the number of open roles,
|
||||
and every open role that is saved or voted up by an enabled agent,
|
||||
whatever its rank; given `uids`, those still without a summary,
|
||||
whatever their rank. A batch asks again when it runs."""
|
||||
where, params = self._summary_scope(uids)
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
f"SELECT r.* FROM roles r WHERE {where} "
|
||||
@@ -1908,17 +1927,15 @@ class Store:
|
||||
# ---------------------------------------------------------------- judge
|
||||
|
||||
def judge_rows(self, agent: Agent, uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
|
||||
"""What `judge` may read for one agent: its best ungated scores of
|
||||
open roles that are no copy, as deep and as low as its `limits` say
|
||||
(`judge_top`, `judge_min`), best first, then the `judge_cv_top`
|
||||
ungated ones closest to its CV (`scores.cv_fit`) that the first set
|
||||
leaves out, closest first, each with some text (`article_text` or
|
||||
`description`). Given `uids`, those of them with
|
||||
text, whatever their rank or state. Each row carries `uid`,
|
||||
`title`, `company`, `place`, `article_text`, `description`, and the
|
||||
stored judgement's `input_key`, `judged_at` and `tries` (NULL, NULL
|
||||
and 0 without one), and `given_up` once `tries` reached
|
||||
`GIVE_UP_AFTER`."""
|
||||
"""What `judge` may read for one agent: the open roles in its cut
|
||||
that are no copy and have some text (`article_text` or
|
||||
`description`), those in by fit first, best fit first, then those in
|
||||
by CV closeness, closest first, then those kept because they are
|
||||
liked or saved. Given `uids`, those of them with text, whatever
|
||||
their rank or state. Each row carries `uid`, `title`, `company`,
|
||||
`place`, `article_text`, `description`, and the stored judgement's
|
||||
`input_key`, `judged_at` and `tries` (NULL, NULL and 0 without one),
|
||||
and `given_up` once `tries` reached `GIVE_UP_AFTER`."""
|
||||
text = "(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"
|
||||
select = ("SELECT r.uid, r.title, r.company, r.place, r.article_text, r.description, "
|
||||
"j.input_key, j.judged_at, COALESCE(j.tries, 0) AS tries, "
|
||||
@@ -1928,23 +1945,13 @@ class Store:
|
||||
return self._rows(
|
||||
f"{select} roles r {joined} WHERE r.uid IN (SELECT value FROM json_each(?)) "
|
||||
f"AND {text} ORDER BY r.uid", (agent.id, json.dumps(list(uids))))
|
||||
ranked = ("FROM scores s JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL "
|
||||
"AND top.dup_of IS NULL WHERE s.agent_id = ? AND s.gated_reason = ''")
|
||||
return self._rows(
|
||||
f"""WITH best AS (SELECT s.uid, s.score {ranked}
|
||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid LIMIT ?),
|
||||
by_score AS (SELECT uid, 0 AS part, score AS rank FROM best
|
||||
WHERE score >= ?),
|
||||
by_cv AS (SELECT s.uid, 1 AS part, s.cv_fit AS rank {ranked}
|
||||
AND s.cv_fit IS NOT NULL
|
||||
AND s.uid NOT IN (SELECT uid FROM by_score)
|
||||
ORDER BY s.cv_fit DESC, top.posted_at DESC, s.uid LIMIT ?)
|
||||
{select} (SELECT * FROM by_score UNION ALL SELECT * FROM by_cv) w
|
||||
JOIN roles r ON r.uid = w.uid {joined}
|
||||
WHERE {text}
|
||||
ORDER BY w.part, w.rank DESC, r.posted_at DESC, r.uid""",
|
||||
(agent.id, agent.limits.judge_top, agent.limits.judge_min, agent.id,
|
||||
agent.limits.judge_cv_top, agent.id))
|
||||
f"""{select} scores s JOIN roles r ON r.uid = s.uid AND r.closed_at IS NULL
|
||||
AND r.dup_of IS NULL
|
||||
{joined}
|
||||
WHERE s.agent_id = ? AND s.cut <> '' AND {text}
|
||||
ORDER BY {_CUT_ORDER}, r.posted_at DESC, r.uid""",
|
||||
(agent.id, agent.id))
|
||||
|
||||
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
|
||||
experience: int, career: int, strengths: Sequence[str],
|
||||
@@ -2031,18 +2038,15 @@ class Store:
|
||||
keeps it."""
|
||||
return bool(self._write("UPDATE roles SET posting_url = ? WHERE uid = ?", (url, uid)))
|
||||
|
||||
# --------------------------------------------------------------- enrich
|
||||
# ----------------------------------------------------------- fetch text
|
||||
|
||||
def enrich_waiting(self, top: int, floor: float,
|
||||
uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
|
||||
"""Open board and search roles without text that an enabled agent with a prompt
|
||||
ranks ungated among its best `top` and at `floor` or above, best
|
||||
first by the highest such score, as `{uid, source_id,
|
||||
subscription_id, title, url, published_at, raw, score}`. The ranking
|
||||
is `_top_scope`'s, with the bar given rather than the agent's own.
|
||||
Given `uids`, those of them still without text, whatever their rank,
|
||||
in no order (`score` is None), since a batch finishes what it was
|
||||
dealt."""
|
||||
def fetch_text_waiting(self, uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
|
||||
"""Open board and search roles without text in the cut of an enabled
|
||||
agent, best first by the highest final score any such agent gives
|
||||
them, as `{uid, source_id, subscription_id, title, url, published_at,
|
||||
raw, score}`. Given `uids`, those of them still without text, cut or
|
||||
not, in no order (`score` is None), since a batch finishes what it
|
||||
was dealt."""
|
||||
fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, "
|
||||
"r.published_at, r.raw")
|
||||
waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source IN ('ats', 'search') "
|
||||
@@ -2054,21 +2058,11 @@ class Store:
|
||||
"AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),))
|
||||
else:
|
||||
rows = self._rows(
|
||||
f"""SELECT {fields}, MAX(ranked.score) AS score FROM (
|
||||
SELECT s.uid, s.score, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.agent_id
|
||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
AND EXISTS (SELECT 1 FROM agent_prompts p
|
||||
WHERE p.agent_id = a.id
|
||||
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
AND top.dup_of IS NULL
|
||||
WHERE s.gated_reason = '') ranked
|
||||
JOIN roles r ON r.uid = ranked.uid
|
||||
WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting}
|
||||
GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor))
|
||||
f"""SELECT {fields}, MAX(s.score) AS score FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
JOIN roles r ON r.uid = s.uid
|
||||
WHERE s.cut <> '' AND {waiting}
|
||||
GROUP BY r.uid ORDER BY score DESC, r.uid""")
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
# -------------------------------------------------------------- vectors
|
||||
@@ -2220,10 +2214,41 @@ class Store:
|
||||
return found
|
||||
|
||||
def delete_agent(self, agent_id: int) -> bool:
|
||||
"""Its prompts, scores, votes and affinities go with it, by the
|
||||
foreign keys."""
|
||||
"""Its prompts, scores, votes, affinities and saved filters go with
|
||||
it, by the foreign keys."""
|
||||
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
|
||||
|
||||
# -------------------------------------------------------- saved filters
|
||||
|
||||
def saved_filters(self, agent_id: int) -> list[SavedFilter]:
|
||||
"""The agent's saved filters, by name regardless of case."""
|
||||
return [SavedFilter.from_row(r) for r in self._rows(
|
||||
"SELECT * FROM saved_filters WHERE agent_id=? ORDER BY name COLLATE NOCASE, id",
|
||||
(agent_id,))]
|
||||
|
||||
def save_filter(self, agent_id: int, name: str, query: str) -> SavedFilter:
|
||||
"""Save the feed's filters under `name` for the agent. ValueError on
|
||||
an empty name or one the agent already has regardless of case,
|
||||
KeyError on an unknown agent."""
|
||||
name = " ".join(name.split())
|
||||
if not name:
|
||||
raise ValueError("a saved filter needs a name")
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
|
||||
raise KeyError(agent_id)
|
||||
new_id = conn.execute(
|
||||
"INSERT INTO saved_filters (agent_id, name, query, created_at) "
|
||||
"VALUES (?,?,?,?)", (agent_id, name, query, self.now())).lastrowid
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"a filter named {name!r} is already saved") from exc
|
||||
row = self._row("SELECT * FROM saved_filters WHERE id=?", (new_id,))
|
||||
assert row is not None
|
||||
return SavedFilter.from_row(row)
|
||||
|
||||
def delete_filter(self, filter_id: int) -> bool:
|
||||
return self._write("DELETE FROM saved_filters WHERE id=?", (filter_id,)) > 0
|
||||
|
||||
def prompt_vector(self, agent_id: int, name: str, key: str) -> np.ndarray | None:
|
||||
"""The cached vector of the agent's prompt `name`, if it was made
|
||||
under `key`."""
|
||||
@@ -2262,11 +2287,11 @@ class Store:
|
||||
|
||||
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
|
||||
"""Every open role of a company this agent has not hidden, copies
|
||||
(`rebuild_dupes`) aside, with the agent's affinity for the company
|
||||
and how the role meets each gate: 1 match, 0 no match, -1 the column
|
||||
is empty. The gates are evaluated here, in SQL, so a `require` gate is
|
||||
the `WHERE` it reads as, written
|
||||
as a column so a gated role keeps its row and reason."""
|
||||
(`rebuild_dupes`) aside, with the agent's affinity for the company,
|
||||
the company's All countries and All roles, and how the role meets
|
||||
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
|
||||
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
|
||||
as, written as a column so a gated role keeps its row and reason."""
|
||||
columns: list[str] = []
|
||||
params: list[Any] = []
|
||||
gaz = self.gazetteer()
|
||||
@@ -2281,31 +2306,41 @@ class Store:
|
||||
params += values
|
||||
columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}")
|
||||
select = ", ".join(["r.uid", "r.posted_at",
|
||||
"COALESCE(ac.affinity, 0) AS affinity", *columns])
|
||||
"COALESCE(ac.affinity, 0) AS affinity",
|
||||
"COALESCE(c.all_countries, 0) AS all_countries",
|
||||
"COALESCE(c.all_roles, 0) AS all_roles", *columns])
|
||||
rows = self._rows(
|
||||
f"""SELECT {select} FROM roles r
|
||||
LEFT JOIN agent_companies ac
|
||||
ON ac.company_key = r.company_key AND ac.agent_id = ?
|
||||
LEFT JOIN companies c ON c.key = r.company_key
|
||||
WHERE r.closed_at IS NULL AND r.dup_of IS NULL
|
||||
AND COALESCE(ac.affinity, 0) <> -1""",
|
||||
[*params, agent_id])
|
||||
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
|
||||
affinity=int(r["affinity"]),
|
||||
all_countries=bool(r["all_countries"]), all_roles=bool(r["all_roles"]),
|
||||
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
||||
for r in rows]
|
||||
|
||||
def saved_uids(self) -> set[str]:
|
||||
"""The roles saved (an application names them), which make every
|
||||
agent's cut whatever their levels."""
|
||||
return {r["role_uid"] for r in self._rows(
|
||||
"SELECT DISTINCT role_uid FROM applications WHERE role_uid IS NOT NULL")}
|
||||
|
||||
def replace_scores(self, agent_id: int,
|
||||
rows: Sequence[tuple[str, float, str] | tuple[str, float, str, str]
|
||||
| tuple[str, float, str, str, float | None]]
|
||||
) -> int:
|
||||
rows: Sequence[tuple[Any, ...]]) -> int:
|
||||
"""`(uid, score, gated_reason)` for every role this agent scored,
|
||||
with the name of the prompt that gave the fit as a fourth item ('' when
|
||||
left out) and the role's closeness to the agent's CV as a fifth (None
|
||||
when left out), replacing whatever it had: a role closed or muted since the last run
|
||||
drops out rather than keeping a stale score. One transaction, so a
|
||||
reader sees the old list or the new one. An agent deleted while its
|
||||
run was scoring gets nothing written and 0 back: its rows are gone
|
||||
with it, and an insert would only fail the foreign key."""
|
||||
and optionally, in this order, the name of the prompt that gave the
|
||||
fit ('' when left out), the role's closeness to the agent's CV, its
|
||||
prompt fit level (both None when left out) and why it is in the cut
|
||||
('' when left out: not in it). Replaces whatever the agent had: a
|
||||
role closed or muted since the last run drops out rather than keeping
|
||||
a stale score. One transaction, so a reader sees the old list or the
|
||||
new one. An agent deleted while its run was scoring gets nothing
|
||||
written and 0 back: its rows are gone with it, and an insert would
|
||||
only fail the foreign key."""
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
|
||||
@@ -2313,13 +2348,20 @@ class Store:
|
||||
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
|
||||
conn.executemany(
|
||||
"""INSERT INTO scores (uid, agent_id, score, gated_reason, prompt, cv_fit,
|
||||
scored_at)
|
||||
VALUES (?,?,?,?,?,?,?)""",
|
||||
fit, cut, scored_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?)""",
|
||||
[(row[0], agent_id, row[1], row[2], row[3] if len(row) > 3 else "",
|
||||
row[4] if len(row) > 4 else None, stamp)
|
||||
row[4] if len(row) > 4 else None, row[5] if len(row) > 5 else None,
|
||||
row[6] if len(row) > 6 else "", stamp)
|
||||
for row in rows])
|
||||
return len(rows)
|
||||
|
||||
def cut(self, agent_id: int) -> dict[str, str]:
|
||||
"""`uid -> why` for the roles in one agent's cut: 'fit', 'cv' or
|
||||
'kept'."""
|
||||
return {r["uid"]: str(r["cut"]) for r in self._rows(
|
||||
"SELECT uid, cut FROM scores WHERE agent_id=? AND cut <> ''", (agent_id,))}
|
||||
|
||||
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
|
||||
"""`uid -> (score, gated_reason)` for one agent."""
|
||||
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
|
||||
|
||||
Reference in new issue
Block a user