Rank by the cut: fetch gate overrides in scoring, the cut per agent, stages read the cut, fetch text renamed from enrich, the feed kept to the fetch gate, and migration 32
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
060ccf66e6
commit
617356e60c
35 files changed
+1054
-751
No files matched your search
+167
-125
@@ -25,7 +25,7 @@ from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
|
||||
RoleCopy, RolePlace, RoleRow, ScoredRole, cv_languages as normal_languages, cv_score,
|
||||
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
|
||||
BROKEN_CAP, capped_career, language_flag, role_languages,
|
||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
|
||||
@@ -42,7 +42,7 @@ _RAW_TEXT = ("place", "currency")
|
||||
#: The host's own words a reader hands on in `Item.raw`, stored as
|
||||
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
|
||||
#: rewritten every pull, as the host's level always was; the others are kept
|
||||
#: filled, since a role's own page (`enrich`) may state what its listing
|
||||
#: filled, since a role's own page (`fetch_text`) may state what its listing
|
||||
#: does not.
|
||||
_HOST_WORDS = ("seniority", "employment", "remote", "country")
|
||||
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
|
||||
@@ -78,6 +78,16 @@ _EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_whe
|
||||
"OR languages_required IS NULL) "
|
||||
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
#: A role in the cut of an enabled agent (`scores.cut`, see `app.jobs.score`),
|
||||
#: as a `WHERE` on `r`: what fetch text and extract read.
|
||||
_IN_CUT = ("r.uid IN (SELECT s.uid FROM scores s JOIN agents a ON a.id = s.agent_id "
|
||||
"AND a.enabled = 1 WHERE s.cut <> '')")
|
||||
|
||||
#: The order within one agent's cut, over its scores as `s`: in by fit, best
|
||||
#: fit first, then in by CV closeness, closest first, then kept.
|
||||
_CUT_ORDER = ("CASE s.cut WHEN 'fit' THEN 0 WHEN 'cv' THEN 1 ELSE 2 END, "
|
||||
"CASE s.cut WHEN 'fit' THEN s.fit WHEN 'cv' THEN s.cv_fit END DESC")
|
||||
|
||||
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
||||
VOTES = (-1, 0, 1, None)
|
||||
|
||||
@@ -1127,7 +1137,7 @@ class Store:
|
||||
hide_muted: bool = False, include_read: bool = True,
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, pay: str | None = None,
|
||||
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
|
||||
company_keys: Sequence[str] = (), places: Sequence[str] = (),
|
||||
seniority: Sequence[str] = (), seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False, by_cv: bool = False, closing: bool = False,
|
||||
@@ -1172,7 +1182,9 @@ class Store:
|
||||
set. The score is the stored one moved by the agent's vote
|
||||
(`_VOTED_SCORE`), so a vote shows in the order at once and needs no
|
||||
rescore. With `by_score` off the order is newest, with the votes and
|
||||
no score.
|
||||
no score; with `fetch_gate` as well, the newest order keeps to the
|
||||
agent's fetch gate the same way, carrying `gated_reason`, so no order
|
||||
shows a role the agent's gate rejects unless `gated`.
|
||||
|
||||
With `by_cv` as well, the roles the agent's `judge` read come first
|
||||
by `cv_score`, then the others by how close they are to the CV
|
||||
@@ -1191,6 +1203,7 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
fetch_gate=fetch_gate,
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
||||
closing=closing)
|
||||
@@ -1202,7 +1215,9 @@ class Store:
|
||||
if scored:
|
||||
select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt, s.cv_fit"
|
||||
order = (_by_cv(cv) if by_cv else "") + "score DESC, " + order
|
||||
elif open_only:
|
||||
elif agent is not None and fetch_gate:
|
||||
select += ", s.gated_reason"
|
||||
if not scored and open_only:
|
||||
# Newest first, open only: idx_roles_posted covers this exactly
|
||||
# (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
|
||||
# LIMIT rather than sorting every open role first. Left to the
|
||||
@@ -1241,7 +1256,7 @@ class Store:
|
||||
hide_muted: bool = False, include_read: bool = True,
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, pay: str | None = None,
|
||||
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
|
||||
company_keys: Sequence[str] = (), places: Sequence[str] = (),
|
||||
seniority: Sequence[str] = (), seniority_unknown: bool = True,
|
||||
pay_unknown: bool = False, by_cv: bool = False, closing: bool = False,
|
||||
@@ -1264,6 +1279,7 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
fetch_gate=fetch_gate,
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
||||
closing=closing)
|
||||
@@ -1280,7 +1296,9 @@ class Store:
|
||||
if scored:
|
||||
select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt, s.cv_fit"
|
||||
order = (_by_cv(cv) if by_cv else "") + "score DESC, " + order
|
||||
elif open_only:
|
||||
elif agent is not None and fetch_gate:
|
||||
select += ", s.gated_reason"
|
||||
if not scored and open_only:
|
||||
# Same reasoning as `roles`: the planner otherwise sorts every
|
||||
# open role before applying LIMIT.
|
||||
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
|
||||
@@ -1318,7 +1336,8 @@ class Store:
|
||||
days: int | None = None, hide_muted: bool = False,
|
||||
include_read: bool = True, tags: Sequence[str] = (),
|
||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||
gated: bool = False, judged: str | None = None,
|
||||
gated: bool = False, fetch_gate: bool = False,
|
||||
judged: str | None = None,
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True, pay_unknown: bool = False,
|
||||
@@ -1328,6 +1347,7 @@ class Store:
|
||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
fetch_gate=fetch_gate,
|
||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
||||
closing=closing)
|
||||
@@ -1340,7 +1360,8 @@ class Store:
|
||||
days: int | None = None, hide_muted: bool = False,
|
||||
include_read: bool = True, tags: Sequence[str] = (),
|
||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||
gated: bool = False, judged: str | None = None,
|
||||
gated: bool = False, fetch_gate: bool = False,
|
||||
judged: str | None = None,
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True,
|
||||
@@ -1356,7 +1377,7 @@ class Store:
|
||||
"company": company, "place": place, "open_only": open_only, "q": q,
|
||||
"function": function, "remote": remote, "days": days, "hide_muted": hide_muted,
|
||||
"include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent,
|
||||
"by_score": by_score, "gated": gated, "judged": judged,
|
||||
"by_score": by_score, "gated": gated, "fetch_gate": fetch_gate, "judged": judged,
|
||||
"company_keys": company_keys, "places": places, "closing": closing}
|
||||
by_level, level_params = self._roles_from(
|
||||
**shared, pay=pay, pay_unknown=pay_unknown)
|
||||
@@ -1376,6 +1397,7 @@ class Store:
|
||||
days: int | None, hide_muted: bool, include_read: bool,
|
||||
tags: Sequence[str], tag_mode: str, agent: int | None,
|
||||
by_score: bool, gated: bool, judged: str | None,
|
||||
fetch_gate: bool = False,
|
||||
pay: str | None = None, company_keys: Sequence[str] = (),
|
||||
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
||||
seniority_unknown: bool = True,
|
||||
@@ -1384,7 +1406,9 @@ class Store:
|
||||
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
||||
its parameters in order. The agent's votes are always joined as `v`
|
||||
(see `_WITH_VOTES`) and its judgements as `j` (`_WITH_JUDGEMENTS`);
|
||||
its scores as `s` only when ordering by them. A copy of another role
|
||||
its scores as `s` when ordering by them or with `fetch_gate`, and
|
||||
then a role the agent's fetch gate rejects is left out unless
|
||||
`gated`. A copy of another role
|
||||
(`rebuild_dupes`) is never listed: its kept role stands for it and
|
||||
counts its copies. `closing` keeps the roles whose deadline is
|
||||
today or within `CLOSING_DAYS` after it."""
|
||||
@@ -1458,7 +1482,7 @@ class Store:
|
||||
params += values
|
||||
sql = f"FROM roles {_WITH_VOTES} {_WITH_JUDGEMENTS}"
|
||||
joins: list[Any] = [agent, agent]
|
||||
if agent is not None and by_score:
|
||||
if agent is not None and (by_score or fetch_gate):
|
||||
sql += " JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?"
|
||||
joins.append(agent)
|
||||
if not gated:
|
||||
@@ -1639,21 +1663,18 @@ class Store:
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` fragment for a role among the best ungated scores of
|
||||
an enabled agent, as deep and as low as that agent's own `limits`
|
||||
say for `stage`: what the two model stages, extract and summarise,
|
||||
are bounded to. The rank is numbered per agent and the agent's
|
||||
limits ride out of the window subquery beside it, so each row is
|
||||
held to the limits of the agent that ranked it; a key the JSON lacks
|
||||
is the `Limits` default. The role is `r`. A company an agent hid
|
||||
needs no clause: `score` writes no rows for it under that agent. An
|
||||
agent without a prompt with text counts for nothing here: its scores
|
||||
have no fit, so they say how fresh a role is, not whether it is
|
||||
wanted."""
|
||||
defaults = Limits()
|
||||
return (f"""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
|
||||
def _summarise_top(self) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` fragment for a role among the best `summarise_top`
|
||||
final scores of an enabled agent with a prompt, open, no copy and
|
||||
passing that agent's fetch gate: the top of its feed. The rank is
|
||||
numbered per agent and the agent's limits ride out of the window
|
||||
subquery beside it, so each row is held to the limit of the agent
|
||||
that ranked it; a key the JSON lacks is the `Limits` default. The
|
||||
role is `r`. An agent without a prompt with text counts for nothing
|
||||
here: its scores have no fit, so they say how fresh a role is, not
|
||||
whether it is wanted."""
|
||||
return ("""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, a.limits, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.agent_id
|
||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
@@ -1664,15 +1685,14 @@ class Store:
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
AND top.dup_of IS NULL
|
||||
WHERE s.gated_reason = '')
|
||||
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
|
||||
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
|
||||
[getattr(defaults, f"{stage}_top"), getattr(defaults, f"{stage}_min")])
|
||||
WHERE n <= COALESCE(json_extract(limits, '$.summarise_top'), ?))""",
|
||||
[Limits().summarise_top])
|
||||
|
||||
def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` for a role `extract` may read: open, never answered,
|
||||
not given up. Given `uids`, those of them and nothing else, since a
|
||||
batch asks again when it runs and must finish what it was dealt;
|
||||
otherwise only the roles in `_top_scope`."""
|
||||
otherwise only the roles in the cut of an enabled agent (`_IN_CUT`)."""
|
||||
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
|
||||
.replace("dup_of", "r.dup_of")
|
||||
@@ -1682,8 +1702,7 @@ class Store:
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
top, params = self._top_scope("extract")
|
||||
return " AND ".join([*where, top]), params
|
||||
return " AND ".join([*where, _IN_CUT]), []
|
||||
|
||||
def extract_pending(self) -> int:
|
||||
where, params = self._extract_scope(None)
|
||||
@@ -1691,14 +1710,13 @@ class Store:
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `extract` has yet to read: the best of every enabled agent's
|
||||
scores, as deep and as low as that agent's own `limits` say, so the
|
||||
backlog follows the rankings rather than the boards. Ordered by the
|
||||
best rank any agent gives the role (each agent's first, then each
|
||||
agent's second ...), then by its score, so every agent's top comes
|
||||
before anyone's tail whatever limits cut the batch. Given `uids`,
|
||||
only those of them still waiting, whatever their rank; a role no agent
|
||||
ranks comes last."""
|
||||
"""Roles `extract` has yet to read: those in the cut of an enabled
|
||||
agent, so the backlog follows the cut rather than the boards.
|
||||
Ordered by the best rank any agent gives the role by final score
|
||||
(each agent's first, then each agent's second ...), then by that
|
||||
score, so every agent's top comes before anyone's tail whatever
|
||||
limit cuts the batch. Given `uids`, only those of them still waiting,
|
||||
whatever their rank; a role no agent ranks comes last."""
|
||||
where, params = self._extract_scope(uids)
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
"SELECT r.* FROM roles r LEFT JOIN ("
|
||||
@@ -1826,15 +1844,16 @@ class Store:
|
||||
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
|
||||
some text to summarise. Given `uids`, those of them and nothing else,
|
||||
open or not and given up or not, because somebody asked for them.
|
||||
Otherwise open, not given up, and in `_top_scope` or one the owner
|
||||
cares about whatever its rank: saved (it has an application) or voted
|
||||
up by an enabled agent."""
|
||||
Otherwise open, not given up, and in the top of an enabled agent's
|
||||
feed (`_summarise_top`) or one the owner cares about whatever its
|
||||
rank: saved (it has an application) or voted up by an enabled
|
||||
agent."""
|
||||
where = ["r.summary = ''",
|
||||
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
top, params = self._top_scope("summarise")
|
||||
top, params = self._summarise_top()
|
||||
where += [
|
||||
"r.closed_at IS NULL",
|
||||
"r.dup_of IS NULL",
|
||||
@@ -1856,12 +1875,12 @@ class Store:
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `summarise` has yet to write, newest first: the top of every
|
||||
enabled agent's scores, as deep and as low as that agent's own
|
||||
`limits` say, so the backlog is bounded by the agents rather than by
|
||||
the number of open roles, and every open role that is saved or voted
|
||||
up by an enabled agent, whatever its rank; given `uids`, those still without a
|
||||
summary, whatever their rank. A batch asks again when it runs."""
|
||||
"""Roles `summarise` has yet to write, newest first: the best
|
||||
`summarise_top` final scores of every enabled agent, so the backlog
|
||||
is bounded by the agents rather than by the number of open roles,
|
||||
and every open role that is saved or voted up by an enabled agent,
|
||||
whatever its rank; given `uids`, those still without a summary,
|
||||
whatever their rank. A batch asks again when it runs."""
|
||||
where, params = self._summary_scope(uids)
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
f"SELECT r.* FROM roles r WHERE {where} "
|
||||
@@ -1908,17 +1927,15 @@ class Store:
|
||||
# ---------------------------------------------------------------- judge
|
||||
|
||||
def judge_rows(self, agent: Agent, uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
|
||||
"""What `judge` may read for one agent: its best ungated scores of
|
||||
open roles that are no copy, as deep and as low as its `limits` say
|
||||
(`judge_top`, `judge_min`), best first, then the `judge_cv_top`
|
||||
ungated ones closest to its CV (`scores.cv_fit`) that the first set
|
||||
leaves out, closest first, each with some text (`article_text` or
|
||||
`description`). Given `uids`, those of them with
|
||||
text, whatever their rank or state. Each row carries `uid`,
|
||||
`title`, `company`, `place`, `article_text`, `description`, and the
|
||||
stored judgement's `input_key`, `judged_at` and `tries` (NULL, NULL
|
||||
and 0 without one), and `given_up` once `tries` reached
|
||||
`GIVE_UP_AFTER`."""
|
||||
"""What `judge` may read for one agent: the open roles in its cut
|
||||
that are no copy and have some text (`article_text` or
|
||||
`description`), those in by fit first, best fit first, then those in
|
||||
by CV closeness, closest first, then those kept because they are
|
||||
liked or saved. Given `uids`, those of them with text, whatever
|
||||
their rank or state. Each row carries `uid`, `title`, `company`,
|
||||
`place`, `article_text`, `description`, and the stored judgement's
|
||||
`input_key`, `judged_at` and `tries` (NULL, NULL and 0 without one),
|
||||
and `given_up` once `tries` reached `GIVE_UP_AFTER`."""
|
||||
text = "(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"
|
||||
select = ("SELECT r.uid, r.title, r.company, r.place, r.article_text, r.description, "
|
||||
"j.input_key, j.judged_at, COALESCE(j.tries, 0) AS tries, "
|
||||
@@ -1928,23 +1945,13 @@ class Store:
|
||||
return self._rows(
|
||||
f"{select} roles r {joined} WHERE r.uid IN (SELECT value FROM json_each(?)) "
|
||||
f"AND {text} ORDER BY r.uid", (agent.id, json.dumps(list(uids))))
|
||||
ranked = ("FROM scores s JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL "
|
||||
"AND top.dup_of IS NULL WHERE s.agent_id = ? AND s.gated_reason = ''")
|
||||
return self._rows(
|
||||
f"""WITH best AS (SELECT s.uid, s.score {ranked}
|
||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid LIMIT ?),
|
||||
by_score AS (SELECT uid, 0 AS part, score AS rank FROM best
|
||||
WHERE score >= ?),
|
||||
by_cv AS (SELECT s.uid, 1 AS part, s.cv_fit AS rank {ranked}
|
||||
AND s.cv_fit IS NOT NULL
|
||||
AND s.uid NOT IN (SELECT uid FROM by_score)
|
||||
ORDER BY s.cv_fit DESC, top.posted_at DESC, s.uid LIMIT ?)
|
||||
{select} (SELECT * FROM by_score UNION ALL SELECT * FROM by_cv) w
|
||||
JOIN roles r ON r.uid = w.uid {joined}
|
||||
WHERE {text}
|
||||
ORDER BY w.part, w.rank DESC, r.posted_at DESC, r.uid""",
|
||||
(agent.id, agent.limits.judge_top, agent.limits.judge_min, agent.id,
|
||||
agent.limits.judge_cv_top, agent.id))
|
||||
f"""{select} scores s JOIN roles r ON r.uid = s.uid AND r.closed_at IS NULL
|
||||
AND r.dup_of IS NULL
|
||||
{joined}
|
||||
WHERE s.agent_id = ? AND s.cut <> '' AND {text}
|
||||
ORDER BY {_CUT_ORDER}, r.posted_at DESC, r.uid""",
|
||||
(agent.id, agent.id))
|
||||
|
||||
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
|
||||
experience: int, career: int, strengths: Sequence[str],
|
||||
@@ -2031,18 +2038,15 @@ class Store:
|
||||
keeps it."""
|
||||
return bool(self._write("UPDATE roles SET posting_url = ? WHERE uid = ?", (url, uid)))
|
||||
|
||||
# --------------------------------------------------------------- enrich
|
||||
# ----------------------------------------------------------- fetch text
|
||||
|
||||
def enrich_waiting(self, top: int, floor: float,
|
||||
uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
|
||||
"""Open board and search roles without text that an enabled agent with a prompt
|
||||
ranks ungated among its best `top` and at `floor` or above, best
|
||||
first by the highest such score, as `{uid, source_id,
|
||||
subscription_id, title, url, published_at, raw, score}`. The ranking
|
||||
is `_top_scope`'s, with the bar given rather than the agent's own.
|
||||
Given `uids`, those of them still without text, whatever their rank,
|
||||
in no order (`score` is None), since a batch finishes what it was
|
||||
dealt."""
|
||||
def fetch_text_waiting(self, uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
|
||||
"""Open board and search roles without text in the cut of an enabled
|
||||
agent, best first by the highest final score any such agent gives
|
||||
them, as `{uid, source_id, subscription_id, title, url, published_at,
|
||||
raw, score}`. Given `uids`, those of them still without text, cut or
|
||||
not, in no order (`score` is None), since a batch finishes what it
|
||||
was dealt."""
|
||||
fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, "
|
||||
"r.published_at, r.raw")
|
||||
waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source IN ('ats', 'search') "
|
||||
@@ -2054,21 +2058,11 @@ class Store:
|
||||
"AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),))
|
||||
else:
|
||||
rows = self._rows(
|
||||
f"""SELECT {fields}, MAX(ranked.score) AS score FROM (
|
||||
SELECT s.uid, s.score, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.agent_id
|
||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
AND EXISTS (SELECT 1 FROM agent_prompts p
|
||||
WHERE p.agent_id = a.id
|
||||
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
AND top.dup_of IS NULL
|
||||
WHERE s.gated_reason = '') ranked
|
||||
JOIN roles r ON r.uid = ranked.uid
|
||||
WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting}
|
||||
GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor))
|
||||
f"""SELECT {fields}, MAX(s.score) AS score FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
JOIN roles r ON r.uid = s.uid
|
||||
WHERE s.cut <> '' AND {waiting}
|
||||
GROUP BY r.uid ORDER BY score DESC, r.uid""")
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
# -------------------------------------------------------------- vectors
|
||||
@@ -2220,10 +2214,41 @@ class Store:
|
||||
return found
|
||||
|
||||
def delete_agent(self, agent_id: int) -> bool:
|
||||
"""Its prompts, scores, votes and affinities go with it, by the
|
||||
foreign keys."""
|
||||
"""Its prompts, scores, votes, affinities and saved filters go with
|
||||
it, by the foreign keys."""
|
||||
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
|
||||
|
||||
# -------------------------------------------------------- saved filters
|
||||
|
||||
def saved_filters(self, agent_id: int) -> list[SavedFilter]:
|
||||
"""The agent's saved filters, by name regardless of case."""
|
||||
return [SavedFilter.from_row(r) for r in self._rows(
|
||||
"SELECT * FROM saved_filters WHERE agent_id=? ORDER BY name COLLATE NOCASE, id",
|
||||
(agent_id,))]
|
||||
|
||||
def save_filter(self, agent_id: int, name: str, query: str) -> SavedFilter:
|
||||
"""Save the feed's filters under `name` for the agent. ValueError on
|
||||
an empty name or one the agent already has regardless of case,
|
||||
KeyError on an unknown agent."""
|
||||
name = " ".join(name.split())
|
||||
if not name:
|
||||
raise ValueError("a saved filter needs a name")
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
|
||||
raise KeyError(agent_id)
|
||||
new_id = conn.execute(
|
||||
"INSERT INTO saved_filters (agent_id, name, query, created_at) "
|
||||
"VALUES (?,?,?,?)", (agent_id, name, query, self.now())).lastrowid
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"a filter named {name!r} is already saved") from exc
|
||||
row = self._row("SELECT * FROM saved_filters WHERE id=?", (new_id,))
|
||||
assert row is not None
|
||||
return SavedFilter.from_row(row)
|
||||
|
||||
def delete_filter(self, filter_id: int) -> bool:
|
||||
return self._write("DELETE FROM saved_filters WHERE id=?", (filter_id,)) > 0
|
||||
|
||||
def prompt_vector(self, agent_id: int, name: str, key: str) -> np.ndarray | None:
|
||||
"""The cached vector of the agent's prompt `name`, if it was made
|
||||
under `key`."""
|
||||
@@ -2262,11 +2287,11 @@ class Store:
|
||||
|
||||
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
|
||||
"""Every open role of a company this agent has not hidden, copies
|
||||
(`rebuild_dupes`) aside, with the agent's affinity for the company
|
||||
and how the role meets each gate: 1 match, 0 no match, -1 the column
|
||||
is empty. The gates are evaluated here, in SQL, so a `require` gate is
|
||||
the `WHERE` it reads as, written
|
||||
as a column so a gated role keeps its row and reason."""
|
||||
(`rebuild_dupes`) aside, with the agent's affinity for the company,
|
||||
the company's All countries and All roles, and how the role meets
|
||||
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
|
||||
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
|
||||
as, written as a column so a gated role keeps its row and reason."""
|
||||
columns: list[str] = []
|
||||
params: list[Any] = []
|
||||
gaz = self.gazetteer()
|
||||
@@ -2281,31 +2306,41 @@ class Store:
|
||||
params += values
|
||||
columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}")
|
||||
select = ", ".join(["r.uid", "r.posted_at",
|
||||
"COALESCE(ac.affinity, 0) AS affinity", *columns])
|
||||
"COALESCE(ac.affinity, 0) AS affinity",
|
||||
"COALESCE(c.all_countries, 0) AS all_countries",
|
||||
"COALESCE(c.all_roles, 0) AS all_roles", *columns])
|
||||
rows = self._rows(
|
||||
f"""SELECT {select} FROM roles r
|
||||
LEFT JOIN agent_companies ac
|
||||
ON ac.company_key = r.company_key AND ac.agent_id = ?
|
||||
LEFT JOIN companies c ON c.key = r.company_key
|
||||
WHERE r.closed_at IS NULL AND r.dup_of IS NULL
|
||||
AND COALESCE(ac.affinity, 0) <> -1""",
|
||||
[*params, agent_id])
|
||||
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
|
||||
affinity=int(r["affinity"]),
|
||||
all_countries=bool(r["all_countries"]), all_roles=bool(r["all_roles"]),
|
||||
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
||||
for r in rows]
|
||||
|
||||
def saved_uids(self) -> set[str]:
|
||||
"""The roles saved (an application names them), which make every
|
||||
agent's cut whatever their levels."""
|
||||
return {r["role_uid"] for r in self._rows(
|
||||
"SELECT DISTINCT role_uid FROM applications WHERE role_uid IS NOT NULL")}
|
||||
|
||||
def replace_scores(self, agent_id: int,
|
||||
rows: Sequence[tuple[str, float, str] | tuple[str, float, str, str]
|
||||
| tuple[str, float, str, str, float | None]]
|
||||
) -> int:
|
||||
rows: Sequence[tuple[Any, ...]]) -> int:
|
||||
"""`(uid, score, gated_reason)` for every role this agent scored,
|
||||
with the name of the prompt that gave the fit as a fourth item ('' when
|
||||
left out) and the role's closeness to the agent's CV as a fifth (None
|
||||
when left out), replacing whatever it had: a role closed or muted since the last run
|
||||
drops out rather than keeping a stale score. One transaction, so a
|
||||
reader sees the old list or the new one. An agent deleted while its
|
||||
run was scoring gets nothing written and 0 back: its rows are gone
|
||||
with it, and an insert would only fail the foreign key."""
|
||||
and optionally, in this order, the name of the prompt that gave the
|
||||
fit ('' when left out), the role's closeness to the agent's CV, its
|
||||
prompt fit level (both None when left out) and why it is in the cut
|
||||
('' when left out: not in it). Replaces whatever the agent had: a
|
||||
role closed or muted since the last run drops out rather than keeping
|
||||
a stale score. One transaction, so a reader sees the old list or the
|
||||
new one. An agent deleted while its run was scoring gets nothing
|
||||
written and 0 back: its rows are gone with it, and an insert would
|
||||
only fail the foreign key."""
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
|
||||
@@ -2313,13 +2348,20 @@ class Store:
|
||||
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
|
||||
conn.executemany(
|
||||
"""INSERT INTO scores (uid, agent_id, score, gated_reason, prompt, cv_fit,
|
||||
scored_at)
|
||||
VALUES (?,?,?,?,?,?,?)""",
|
||||
fit, cut, scored_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?)""",
|
||||
[(row[0], agent_id, row[1], row[2], row[3] if len(row) > 3 else "",
|
||||
row[4] if len(row) > 4 else None, stamp)
|
||||
row[4] if len(row) > 4 else None, row[5] if len(row) > 5 else None,
|
||||
row[6] if len(row) > 6 else "", stamp)
|
||||
for row in rows])
|
||||
return len(rows)
|
||||
|
||||
def cut(self, agent_id: int) -> dict[str, str]:
|
||||
"""`uid -> why` for the roles in one agent's cut: 'fit', 'cv' or
|
||||
'kept'."""
|
||||
return {r["uid"]: str(r["cut"]) for r in self._rows(
|
||||
"SELECT uid, cut FROM scores WHERE agent_id=? AND cut <> ''", (agent_id,))}
|
||||
|
||||
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
|
||||
"""`uid -> (score, gated_reason)` for one agent."""
|
||||
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
|
||||
|
||||
Reference in new issue
Block a user