Store copies of one posting as roles.dup_of and leave them out of the feed, the counts and the model stages
Migration 18 adds roles.dup_of with a partial index and runs the first rebuild. Store.rebuild_dupes feeds app.dupes.assign the open roles of every (company_key, title_key) with more than one, rewrites dup_of where it changed, clears it on closed roles, and hands read_at and votes to a changed group's kept role. The pull's settle and `app dupes` run it. RoleRow.copies counts a row's open copies; Role.copies lists them and Role.dup_of names a copy's kept role. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
55e353e764
commit
545c932f7b
10 files changed
+490
-41
No files matched your search
+17
@@ -7,6 +7,7 @@
|
||||
app searches add|list|remove
|
||||
app agents list|prompt
|
||||
app stages [extract|summarise on|off]
|
||||
app dupes
|
||||
app db init
|
||||
|
||||
`app job` queues the work on a queue of its own and drains it here, so a
|
||||
@@ -155,6 +156,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
stages.add_argument("name", nargs="?", choices=["extract", "summarise"])
|
||||
stages.add_argument("state", nargs="?", choices=["on", "off"])
|
||||
|
||||
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
return parser
|
||||
@@ -179,6 +182,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return run_agents(args)
|
||||
if args.command == "stages":
|
||||
return run_stages(args)
|
||||
if args.command == "dupes":
|
||||
return run_dupes(args)
|
||||
return init_db(args)
|
||||
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
@@ -626,6 +631,18 @@ def run_stages(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def run_dupes(args: argparse.Namespace) -> int:
|
||||
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
|
||||
now. Prints the groups (kept roles with a copy), the copies the feed
|
||||
hides and the rows whose `dup_of` changed, as JSON."""
|
||||
from app.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
counts = Store(config.database).rebuild_dupes()
|
||||
print(json.dumps(counts._asdict()))
|
||||
return 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from app.storage import LATEST, Store
|
||||
|
||||
+7
-1
@@ -219,6 +219,11 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
A board or search switched off while its batch was in flight had its
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
|
||||
After the closes, `Store.rebuild_dupes` recomputes which open roles are
|
||||
copies of one posting, so a closed kept role hands its group on and the
|
||||
roles this run added join theirs; `copies` is how many open roles it
|
||||
leaves hidden as copies.
|
||||
"""
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
@@ -252,9 +257,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
release_run(store, run_marker)
|
||||
copies = store.rebuild_dupes().copies
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
|
||||
+23
-1
@@ -763,6 +763,18 @@ JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
|
||||
|
||||
# ------------------------------------------------------------------ models
|
||||
|
||||
class RoleCopy(Model):
|
||||
"""One open copy of a kept role, as the reader lists it under "Also
|
||||
posted": where it is listed (`host`, the search provider or the board's
|
||||
ATS) and the link that opens it there."""
|
||||
|
||||
uid: str
|
||||
host: str = ""
|
||||
place: str = ""
|
||||
original_url: str = ""
|
||||
posted_at: str = ""
|
||||
|
||||
|
||||
class Role(Model):
|
||||
"""A row of `roles`: one posting on one board."""
|
||||
|
||||
@@ -819,6 +831,13 @@ class Role(Model):
|
||||
#: The open vocabulary `summarise` read off the posting, sorted. Not a
|
||||
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
|
||||
tags: list[str] = []
|
||||
#: The uid of the kept role when this one is a copy of the same posting
|
||||
#: (`app.dupes`, `Store.rebuild_dupes`), else None. A copy is left out of
|
||||
#: the feed and the counts but still opens by uid.
|
||||
dup_of: str | None = None
|
||||
#: The open copies of this role, oldest first, when it is the kept one;
|
||||
#: empty otherwise. Not a column: filled by `Store.role`.
|
||||
copies: list[RoleCopy] = []
|
||||
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
@@ -895,7 +914,8 @@ class RoleRow(Model):
|
||||
derived as on `Role`. `original_url` is one short string, kept so the
|
||||
`o` key opens a selected row's posting without opening the reader.
|
||||
`seniority` is the title's level until extract reads the posting, ''
|
||||
when neither says."""
|
||||
when neither says. `copies` counts the open copies the feed hides behind
|
||||
this row."""
|
||||
|
||||
uid: str
|
||||
company_key: str = ""
|
||||
@@ -920,6 +940,8 @@ class RoleRow(Model):
|
||||
read_at: str | None = None
|
||||
tags: list[str] = []
|
||||
excerpt: str = ""
|
||||
#: How many open roles are copies of this one (`Role.copies`), 0 when none.
|
||||
copies: int = 0
|
||||
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
|
||||
@@ -92,6 +92,13 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- whose description is only a teaser.
|
||||
article_text TEXT NOT NULL DEFAULT '',
|
||||
article_at TEXT,
|
||||
-- The uid of the kept role when this open role is a copy of one posting
|
||||
-- (app/dupes.py), NULL on the kept role, on a role without copies and on
|
||||
-- every closed role. Written only by Store.rebuild_dupes, at the end of
|
||||
-- every pull; the feed, the counts and the model stages skip a copy.
|
||||
-- Not the last column: SQLite cannot drop a last column that a comment
|
||||
-- precedes, which the migration tests do.
|
||||
dup_of TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
salary_text TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
@@ -292,6 +299,10 @@ CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote);
|
||||
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
|
||||
-- The copies of a kept role, for `Store.role` and the feed's copy counts.
|
||||
-- Partial: all but about 3% of roles are no copy.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
ON roles(dup_of) WHERE dup_of IS NOT NULL;
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -455,6 +466,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
# A Python step: the fold and the title classifier are Python, as in 11.
|
||||
(17, "roles.place_fold for the whole-word place filter, and roles.seniority from "
|
||||
"the title where extract has not read the role", lambda conn: _fold_places(conn)),
|
||||
# A Python step: the first rebuild runs the rule in app/dupes.py.
|
||||
(18, "roles.dup_of: one posting listed several times is one role, the others "
|
||||
"copies of it", lambda conn: _add_dup_of(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -566,6 +580,18 @@ def _fold_places(conn: sqlite3.Connection) -> None:
|
||||
len(folds), len(place_updates), len(level_updates))
|
||||
|
||||
|
||||
def _add_dup_of(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 18's step. Adds `roles.dup_of` and runs the first rebuild
|
||||
(`store.rebuild_dupes`) inside `_apply`'s transaction, so a migrated file
|
||||
hides its copies from the first open rather than from the next pull.
|
||||
Imported here, not at the top: `store` imports this module."""
|
||||
from app.storage.store import rebuild_dupes
|
||||
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN dup_of TEXT")
|
||||
counts = rebuild_dupes(conn)
|
||||
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+160
-26
@@ -14,16 +14,17 @@ from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from typing import Any, Literal, NamedTuple
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
|
||||
from desk.models import Item
|
||||
from app import places
|
||||
from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RoleRow,
|
||||
ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
@@ -50,8 +51,9 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
|
||||
#: never answers is otherwise paid for on every pull for as long as it exists.
|
||||
GIVE_UP_AFTER = 3
|
||||
|
||||
#: A role `extract` has yet to read: never answered, still open, not given up.
|
||||
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
|
||||
#: A role `extract` has yet to read: never answered, still open, no copy of
|
||||
#: another (`rebuild_dupes`), not given up.
|
||||
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL AND dup_of IS NULL "
|
||||
f"AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
||||
@@ -226,6 +228,93 @@ def _int_or_none(value: Any) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
#: The host `app.dupes.Copy.host` and `RoleCopy.host` read: the provider of
|
||||
#: a search's row, else the ATS a board runs on, as `posted_host` reads it.
|
||||
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
|
||||
|
||||
|
||||
class DupeCounts(NamedTuple):
|
||||
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
|
||||
open roles that are a copy, and rows whose `dup_of` it rewrote."""
|
||||
|
||||
groups: int
|
||||
copies: int
|
||||
changed: int
|
||||
|
||||
|
||||
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
||||
"""Recompute `roles.dup_of` from scratch with `app.dupes.assign`, inside
|
||||
the caller's transaction: `Store.rebuild_dupes` and migration 18.
|
||||
|
||||
Only the open roles of a `(company_key, title_key)` with more than one
|
||||
open role can be a copy, a few thousand of the ~34k, so only those are
|
||||
read. A row whose `dup_of` differs from the rule's answer is rewritten,
|
||||
and a `dup_of` left on a closed role, or on one no longer in such a pair,
|
||||
is cleared: a copy never outlives its posting, and a closed kept role
|
||||
hands its group to the next copy here.
|
||||
|
||||
Where a group changed (a member's `dup_of` was rewritten), its kept role
|
||||
takes over what was done to the posting under another uid. The sources
|
||||
are the group's members and the roles they were copies of before, so a
|
||||
closed kept role still hands over. The kept role takes the earliest
|
||||
`read_at` of them when it has none, and each agent's latest vote among
|
||||
them moves to it unless that agent already voted on it; a vote that
|
||||
cannot move stays where it is. An unchanged group is not touched, so
|
||||
marking a kept role unread again sticks. Applications keep their
|
||||
`role_uid`, and scores of a copy go at the agent's next `score`."""
|
||||
rows = conn.execute(
|
||||
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
||||
description != '' AS described, posted_at
|
||||
FROM roles
|
||||
WHERE closed_at IS NULL AND company_key != ''
|
||||
AND (company_key, title_key) IN (
|
||||
SELECT company_key, title_key FROM roles
|
||||
WHERE closed_at IS NULL AND company_key != ''
|
||||
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
|
||||
new = dupes.assign(
|
||||
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
||||
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None)
|
||||
for r in rows)
|
||||
old: dict[str, str | None] = dict.fromkeys(new)
|
||||
old.update((r[0], r[1]) for r in conn.execute(
|
||||
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
|
||||
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
|
||||
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
|
||||
|
||||
changed = {uid for _, uid in updates}
|
||||
members: dict[str, set[str]] = {}
|
||||
for uid, dup in new.items():
|
||||
members.setdefault(dup or uid, set()).add(uid)
|
||||
groups = {kept: group for kept, group in members.items() if len(group) > 1}
|
||||
for kept, group in groups.items():
|
||||
if not group & changed:
|
||||
continue
|
||||
sources = (group | {old.get(uid) or uid for uid in group}) - {kept}
|
||||
_carry(conn, kept, sources)
|
||||
return DupeCounts(groups=len(groups), copies=sum(dup is not None for dup in new.values()),
|
||||
changed=len(updates))
|
||||
|
||||
|
||||
def _carry(conn: sqlite3.Connection, kept: str, sources: set[str]) -> None:
|
||||
"""`rebuild_dupes`' hand-over to a group's kept role: the earliest
|
||||
`read_at` of `sources` if it has none, and each agent's latest vote among
|
||||
them unless it has one of its own."""
|
||||
wanted = json.dumps(sorted(sources))
|
||||
conn.execute(
|
||||
"""UPDATE roles SET read_at = (SELECT MIN(read_at) FROM roles
|
||||
WHERE uid IN (SELECT value FROM json_each(?)))
|
||||
WHERE uid = ? AND read_at IS NULL""", (wanted, kept))
|
||||
voted = {r[0] for r in conn.execute("SELECT agent_id FROM role_votes WHERE uid = ?", (kept,))}
|
||||
moves: dict[int, str] = {}
|
||||
for agent_id, uid in conn.execute(
|
||||
"""SELECT agent_id, uid FROM role_votes WHERE uid IN (SELECT value FROM json_each(?))
|
||||
ORDER BY voted_at, uid""", (wanted,)):
|
||||
if agent_id not in voted:
|
||||
moves[agent_id] = uid # the latest vote wins, by the order above
|
||||
conn.executemany("UPDATE role_votes SET uid = ? WHERE agent_id = ? AND uid = ?",
|
||||
[(kept, agent_id, uid) for agent_id, uid in moves.items()])
|
||||
|
||||
|
||||
class _StateMap(MutableMapping[str, str]):
|
||||
"""`Store.state`: the key/value table, read and written one key at a time."""
|
||||
|
||||
@@ -443,8 +532,39 @@ class Store:
|
||||
return None
|
||||
role = ScoredRole.from_row(row)
|
||||
role.tags = self._tags_of(uid)
|
||||
role.copies = self._copies_of(uid)
|
||||
return role
|
||||
|
||||
def _copies_of(self, uid: str) -> list[RoleCopy]:
|
||||
"""The open copies of one kept role, oldest first; empty for a copy
|
||||
or a role without any. See `Role.copies`."""
|
||||
return [RoleCopy.from_row(r) for r in self._rows(
|
||||
f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles
|
||||
WHERE dup_of = ? AND closed_at IS NULL ORDER BY posted_at, uid""", (uid,))]
|
||||
|
||||
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
|
||||
"""`uid -> open copies` for a page of kept roles, one grouped query
|
||||
over `idx_roles_dup_of`. A role without copies is absent. The index
|
||||
is named because the planner otherwise takes `idx_roles_closed` for
|
||||
`closed_at IS NULL` and reads every open role, 19 ms against 0.1 on
|
||||
the live database."""
|
||||
if not uids:
|
||||
return {}
|
||||
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
|
||||
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
|
||||
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
|
||||
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
||||
|
||||
def rebuild_dupes(self) -> DupeCounts:
|
||||
"""Recompute which open roles are copies of one posting, in one
|
||||
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
|
||||
because it reads before it writes: a deferred transaction that a
|
||||
pull's upsert wrote under in between could not take the write lock
|
||||
and would fail rather than wait."""
|
||||
with self.transaction() as conn:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
return rebuild_dupes(conn)
|
||||
|
||||
def set_read(self, uid: str, read: bool = True) -> Role:
|
||||
"""Mark a role read, or unread again. Stamps `read_at` only the first
|
||||
time — a role already read keeps the moment it was first opened —
|
||||
@@ -677,9 +797,11 @@ class Store:
|
||||
row = dict(r)
|
||||
row["excerpt"] = excerpt(row["excerpt"])
|
||||
found.append(RoleRow.from_row(row))
|
||||
by_uid = self._tags_by_uid([r.uid for r in found])
|
||||
for r in found:
|
||||
r.tags = by_uid.get(r.uid, [])
|
||||
by_uid = self._tags_by_uid([f.uid for f in found])
|
||||
copies = self._copy_counts([f.uid for f in found])
|
||||
for f in found:
|
||||
f.tags = by_uid.get(f.uid, [])
|
||||
f.copies = copies.get(f.uid, 0)
|
||||
return found
|
||||
|
||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||
@@ -749,8 +871,10 @@ class Store:
|
||||
pay_unknown: bool = False) -> tuple[str, list[Any]]:
|
||||
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
||||
its parameters in order. The agent's votes are always joined as `v`
|
||||
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
|
||||
where: list[str] = []
|
||||
(see `_WITH_VOTES`); its scores as `s` only when ordering by them. A
|
||||
copy of another role (`rebuild_dupes`) is never listed: its kept role
|
||||
stands for it and counts its copies."""
|
||||
where: list[str] = ["roles.dup_of IS NULL"]
|
||||
params: list[Any] = []
|
||||
if company:
|
||||
where.append("company LIKE ? ESCAPE '\\'")
|
||||
@@ -831,7 +955,7 @@ class Store:
|
||||
first and then alphabetically. What `/api/filters` serves; the
|
||||
companies `agent_id` hid are left out unless `show_muted`, the same
|
||||
default as `roles`, and with no agent nothing is hidden."""
|
||||
where = "r.closed_at IS NULL"
|
||||
where = "r.closed_at IS NULL AND r.dup_of IS NULL"
|
||||
params: list[Any] = []
|
||||
if not show_muted and agent_id is not None:
|
||||
where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
|
||||
@@ -913,6 +1037,7 @@ class Store:
|
||||
WHERE p.agent_id = a.id
|
||||
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
AND top.dup_of IS NULL
|
||||
WHERE s.gated_reason = '')
|
||||
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
|
||||
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
|
||||
@@ -924,7 +1049,8 @@ class Store:
|
||||
batch asks again when it runs and must finish what it was dealt;
|
||||
otherwise only the roles in `_top_scope`."""
|
||||
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")]
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
|
||||
.replace("dup_of", "r.dup_of")]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
@@ -1033,6 +1159,7 @@ class Store:
|
||||
top, params = self._top_scope("summarise")
|
||||
where += [
|
||||
"r.closed_at IS NULL",
|
||||
"r.dup_of IS NULL",
|
||||
f"""NOT EXISTS (SELECT 1 FROM state
|
||||
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
|
||||
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
|
||||
@@ -1106,11 +1233,12 @@ class Store:
|
||||
|
||||
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
||||
"""Open roles with no vector from `model`, newest first, so a backlog
|
||||
embedded from the top improves what is on screen first."""
|
||||
embedded from the top improves what is on screen first. A copy is
|
||||
not embedded: it is never scored."""
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
"""SELECT r.* FROM roles r
|
||||
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
|
||||
WHERE v.uid IS NULL AND r.closed_at IS NULL
|
||||
WHERE v.uid IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL
|
||||
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
|
||||
|
||||
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
|
||||
@@ -1239,10 +1367,11 @@ class Store:
|
||||
# --------------------------------------------------------------- scores
|
||||
|
||||
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
|
||||
"""Every open role of a company this agent has not hidden, with the
|
||||
agent's affinity for the company and how the role meets each gate: 1
|
||||
match, 0 no match, -1 the column is empty. The gates are evaluated
|
||||
here, in SQL, so a `require` gate is the `WHERE` it reads as, written
|
||||
"""Every open role of a company this agent has not hidden, copies
|
||||
(`rebuild_dupes`) aside, with the agent's affinity for the company
|
||||
and how the role meets each gate: 1 match, 0 no match, -1 the column
|
||||
is empty. The gates are evaluated here, in SQL, so a `require` gate is
|
||||
the `WHERE` it reads as, written
|
||||
as a column so a gated role keeps its row and reason."""
|
||||
columns: list[str] = []
|
||||
params: list[Any] = []
|
||||
@@ -1257,7 +1386,8 @@ class Store:
|
||||
f"""SELECT {select} FROM roles r
|
||||
LEFT JOIN agent_companies ac
|
||||
ON ac.company_key = r.company_key AND ac.agent_id = ?
|
||||
WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""",
|
||||
WHERE r.closed_at IS NULL AND r.dup_of IS NULL
|
||||
AND COALESCE(ac.affinity, 0) <> -1""",
|
||||
[*params, agent_id])
|
||||
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
|
||||
affinity=int(r["affinity"]),
|
||||
@@ -1403,10 +1533,11 @@ class Store:
|
||||
|
||||
def open_role_counts(self) -> dict[str, int]:
|
||||
"""`company_key -> open role count`, for the company list's row
|
||||
count. A key with no open roles is simply absent."""
|
||||
count. A key with no open roles is simply absent. Copies are not
|
||||
counted, so the number is the rows the feed lists for the company."""
|
||||
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
|
||||
"SELECT company_key, COUNT(*) AS n FROM roles "
|
||||
"WHERE closed_at IS NULL GROUP BY company_key")}
|
||||
"WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")}
|
||||
|
||||
def open_companies(self, agent_id: int | None = None
|
||||
) -> list[tuple[str, list[str], int, int]]:
|
||||
@@ -1415,7 +1546,7 @@ class Store:
|
||||
company row's name and then every spelling its roles carry, commonest
|
||||
first, so a search finds a company by any of them. A company
|
||||
`agent_id` hid is left out; `affinity` is that agent's, 0 without
|
||||
one."""
|
||||
one. Copies are not counted, as in the feed."""
|
||||
rows = self._rows(
|
||||
"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n,
|
||||
c.name AS own, COALESCE(ac.affinity, 0) AS affinity
|
||||
@@ -1423,7 +1554,7 @@ class Store:
|
||||
LEFT JOIN companies c ON c.key = r.company_key
|
||||
LEFT JOIN agent_companies ac ON ac.company_key = r.company_key
|
||||
AND ac.agent_id = ?
|
||||
WHERE r.closed_at IS NULL AND r.company_key != ''
|
||||
WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != ''
|
||||
AND COALESCE(ac.affinity, 0) != -1
|
||||
GROUP BY r.company_key, r.company
|
||||
ORDER BY r.company_key, n DESC, r.company""", (agent_id,))
|
||||
@@ -1439,13 +1570,16 @@ class Store:
|
||||
|
||||
def open_places(self) -> list[tuple[str, str, int]]:
|
||||
"""`(place, country, open roles)` for every place line an open role
|
||||
carries, what `places.suggest` reads."""
|
||||
carries, what `places.suggest` reads. Copies are not counted, as in
|
||||
the feed."""
|
||||
return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows(
|
||||
"SELECT place, country, COUNT(*) AS n FROM roles "
|
||||
"WHERE closed_at IS NULL AND place != '' GROUP BY place, country")]
|
||||
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
|
||||
"GROUP BY place, country")]
|
||||
|
||||
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
||||
"""Per subscription id, its open roles counted by `country`."""
|
||||
"""Per subscription id, its open roles counted by `country`, copies
|
||||
included: what a board or a search holds, not what the feed lists."""
|
||||
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
|
||||
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
||||
GROUP BY subscription_id, country""")
|
||||
|
||||
+7
-2
@@ -133,7 +133,9 @@ def role_facets(store: StoreDep, filters: FeedFilters) -> Facets:
|
||||
def get_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole:
|
||||
"""One role, open or closed, for the read page, with `agent`'s vote and
|
||||
score, and the prompt that gave its fit (the default agent's when absent;
|
||||
None when there is no agent)."""
|
||||
None when there is no agent). `copies` lists the open copies of a kept
|
||||
role (`RoleCopy`, for "Also posted"); a copy itself opens too, with
|
||||
`dup_of` naming its kept role."""
|
||||
role = store.role(uid, _agent_id(store, agent))
|
||||
if role is None:
|
||||
raise HTTPException(404, f"no such role: {uid}")
|
||||
@@ -226,7 +228,10 @@ def list_roles(
|
||||
`region:skane`, `place:malmo`, matched on whole folded words) and
|
||||
`seniority` (repeated levels), each several values joined by OR. A role
|
||||
with no level passes the seniority filter unless `seniority_unknown` is
|
||||
false. The parameters are read by `_feed_filters`."""
|
||||
false. The parameters are read by `_feed_filters`.
|
||||
|
||||
A copy of another role (`Store.rebuild_dupes`) is never a row; its kept
|
||||
role's `copies` says how many it stands for."""
|
||||
return RolesPage(items=store.roles_page(limit=limit, offset=offset, **filters),
|
||||
total=store.count_roles(**filters))
|
||||
|
||||
|
||||
Reference in new issue
Block a user