Places are ordered in tiers off the agent's gates (the any group, then the require scope, then the rest), and within a tier by the home the any group gives, since the gazetteer holds no coordinates. GET /api/roles and /api/roles/{uid} carry place_texts, lead_place and lead_url for the roles served; the gates and the dupes rule are unchanged.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2884 lines
155 KiB
Python
2884 lines
155 KiB
Python
"""`Store`: every SQL statement in the application.
|
|
|
|
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
|
|
has no storage package. A connection is opened per call and closed again, so
|
|
one caller's commit can never land inside another's transaction; in WAL mode
|
|
that costs a file handle.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from collections import Counter
|
|
from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence
|
|
from contextlib import closing, contextmanager
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, Literal, NamedTuple
|
|
from uuid import uuid4
|
|
|
|
import numpy as np
|
|
|
|
from desk.models import Item
|
|
from app import dupes, places
|
|
from app.models import (
|
|
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
|
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
|
|
RoleCopy, RolePlace, RoleRow, ScoredRole, cv_languages as normal_languages, cv_score,
|
|
BROKEN_CAP, capped_career, language_flag, role_languages,
|
|
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
|
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
|
|
)
|
|
from app.storage import gazetteer, schema
|
|
|
|
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
|
#: empty value never overwrites a filled column, so an upsert from a board that
|
|
#: says less than `extract` found keeps what `extract` found. `function` is
|
|
#: not here: it is not a board field (see the note by `keep_filled` below).
|
|
#: Nor is `seniority`, which is the title's level unless a board states one.
|
|
_RAW_TEXT = ("place", "currency")
|
|
|
|
#: The host's own words a reader hands on in `Item.raw`, stored as
|
|
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
|
|
#: rewritten every pull, as the host's level always was; the others are kept
|
|
#: filled, since a role's own page (`enrich`) may state what its listing
|
|
#: does not.
|
|
_HOST_WORDS = ("seniority", "employment", "remote", "country")
|
|
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
|
|
#: What prep maps from that a pull rewrites whatever it brings.
|
|
#: `deadline_host` is the host's last day to apply (`applicationDeadline` in
|
|
#: `raw`, Jobnet and Jobbank), as `YYYY-MM-DD`.
|
|
_PREP_OVERWRITTEN = ("title", "seniority_host", "deadline_host")
|
|
_RAW_INT = ("salary_min", "salary_max")
|
|
|
|
#: What `extract` may fill. Only an empty column is written: a board's own
|
|
#: structured field wins over what a model read out of the prose, and the next
|
|
#: upsert from that board would put it back anyway. `function`,
|
|
#: `seniority` and `remote` are not here, for the opposite reason — see
|
|
#: `set_extracted`.
|
|
_EXTRACT_TEXT = ("place", "country", "currency", "permit")
|
|
|
|
#: How many replies may leave a role unanswered before `extract` stops
|
|
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
|
#: never answers is otherwise paid for on every pull for as long as it exists.
|
|
GIVE_UP_AFTER = 3
|
|
|
|
#: A role `extract` has yet to read: still open, no copy of another
|
|
#: (`rebuild_dupes`), not given up, and either never answered, without a
|
|
#: model level, never asked where a remote hire may live
|
|
#: (`remote_where_model` NULL), or never asked for its languages and deadline
|
|
#: (`languages_required` NULL). The last three make `extract` the
|
|
#: afterburner for the level, the area, the languages and the deadline: a
|
|
#: role read before they were asked for is read again, and only fills what
|
|
#: is still empty (`set_extracted`). A reply
|
|
#: that names no level leaves the role waiting and is charged a try
|
|
#: (`extract.apply_batch`), so it ends.
|
|
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL "
|
|
"OR languages_required IS NULL) "
|
|
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
|
|
|
|
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
|
VOTES = (-1, 0, 1, None)
|
|
|
|
#: The `judged` filter of `Store.roles`, as a `WHERE` on the agent's vote,
|
|
#: joined as `v`. A neutral vote is judged, and is neither liked nor passed.
|
|
JUDGED: dict[str, str] = {
|
|
"liked": "v.vote = 1",
|
|
"unjudged": "v.vote IS NULL",
|
|
"passed": "v.vote = -1",
|
|
}
|
|
|
|
#: The feed's Pay filter, `_roles_from`'s `pay`: the SQL that is true for
|
|
#: each choice, over `roles`. `salary_state` (`app.models.Role`) is derived,
|
|
#: not stored, so the filter spells the same rule out in SQL rather than
|
|
#: reading a column: 'normalised' is `salary_min` or `salary_max` set,
|
|
#: 'stated' is `salary_text` set and neither of those is.
|
|
PAY: dict[str, str] = {
|
|
"stated": "(salary_min IS NOT NULL OR salary_max IS NOT NULL OR salary_text != '')",
|
|
"normalised": "(salary_min IS NOT NULL OR salary_max IS NOT NULL)",
|
|
"stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)",
|
|
}
|
|
|
|
#: A role that states no pay at all, what `pay_unknown` lets through the Pay
|
|
#: filter, and the `""` of its facet.
|
|
_NO_PAY = "(salary_min IS NULL AND salary_max IS NULL AND salary_text = '')"
|
|
|
|
#: `salary_state` in SQL, what the Pay facet counts by: 'normalised',
|
|
#: 'stated' (a line and no figure) or ''.
|
|
_PAY_STATE = ("CASE WHEN salary_min IS NOT NULL OR salary_max IS NOT NULL THEN 'normalised' "
|
|
"WHEN salary_text != '' THEN 'stated' ELSE '' END")
|
|
|
|
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
|
|
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
|
|
#: moves the role at once and the stored score stays what `score` computed.
|
|
_VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
|
|
"WHEN 1 THEN 1.2 ELSE 1 END)")
|
|
|
|
#: The agent's votes beside every role. With no agent the join matches
|
|
#: nothing, so `vote` and `voted_at` are NULL: one query shape either way.
|
|
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
|
|
|
#: The agent's `judge` readings beside every role, the same way: `j.skills`,
|
|
#: `j.experience`, `j.career` and `j.broken` are NULL for a role not judged and with no
|
|
#: agent.
|
|
_WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? "
|
|
"AND j.judged_at IS NOT NULL")
|
|
|
|
|
|
|
|
def _cv_sql(weights: Weights | None) -> str:
|
|
"""`cv_score` as SQL for one agent's weights, unrounded (`app.models.cv_score`
|
|
rounds it): NULL for a role not judged, and for all three weights 0. Career
|
|
counts as at most `BROKEN_CAP` when the judgement holds a broken
|
|
must-have (`j.broken`), as `app.models.capped_career` has it. The
|
|
numbers are the model's validated, finite floats, so they are written in
|
|
rather than bound, the select's placeholders being then no concern."""
|
|
w = weights or Weights()
|
|
total = w.cv_skills + w.cv_experience + w.cv_career
|
|
if not total:
|
|
return "NULL"
|
|
career = (f"(CASE WHEN j.broken > 0 AND j.career > {BROKEN_CAP} THEN {BROKEN_CAP} "
|
|
"ELSE j.career END)")
|
|
return (f"(({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience "
|
|
f"+ {w.cv_career!r} * {career}) / {total!r})")
|
|
|
|
|
|
def _by_cv(expression: str) -> str:
|
|
"""The order of CV match: the judged roles by `cv_score` (its SQL
|
|
`expression`), then the rest by how close they are to the CV
|
|
(`scores.cv_fit`), then by score, as `roles` orders them."""
|
|
return (f"{expression} IS NULL, {expression} DESC, "
|
|
"s.cv_fit IS NULL, s.cv_fit DESC, ")
|
|
|
|
#: How many days after today the feed's Closing soon filter reaches.
|
|
CLOSING_DAYS = 4
|
|
|
|
#: What `roles_page` reads for `RoleRow.excerpt`: the summary when there is
|
|
#: one, else the description's opening. A `SUBSTR` of the source column, not
|
|
#: the column itself, so a big `description` never leaves SQLite whole for a
|
|
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
|
|
_EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), "
|
|
"roles.teaser)")
|
|
|
|
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
|
|
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
|
|
#: still has somewhere to land.
|
|
_EXCERPT_RAW = 240
|
|
|
|
_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city",
|
|
"ats_slug", "watch", "careers_opened_at", "hand_reason", "sector", "stage",
|
|
"size_band", "founded", "last_funding", "last_funding_currency",
|
|
"facts_source", "facts_at", "all_roles", "all_countries")
|
|
|
|
#: A company as the store serves it: the row and the asked-for agent's affinity.
|
|
_COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c
|
|
LEFT JOIN agent_companies ac ON ac.company_key = c.key AND ac.agent_id = ?"""
|
|
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
|
|
"show", "target_id")
|
|
|
|
#: A tag's longest stored form, and how many a role may carry. Long enough for
|
|
#: "senior backend engineer" and short enough that the filter row stays a row.
|
|
_MAX_TAG_LEN = 40
|
|
_MAX_TAGS = 12
|
|
|
|
|
|
def _normalise_tags(tags: Sequence[str]) -> list[str]:
|
|
"""Each tag stripped, lowercased, its internal whitespace collapsed to one
|
|
space and cut to `_MAX_TAG_LEN`; empties and repeats dropped, order kept,
|
|
capped at `_MAX_TAGS`. What `set_tags` and `set_summary` store, so the
|
|
filter row never has to trust what a model or a person typed."""
|
|
seen: set[str] = set()
|
|
out: list[str] = []
|
|
for tag in tags:
|
|
cleaned = " ".join(str(tag).split()).lower()[:_MAX_TAG_LEN]
|
|
if not cleaned or cleaned in seen:
|
|
continue
|
|
seen.add(cleaned)
|
|
out.append(cleaned)
|
|
if len(out) >= _MAX_TAGS:
|
|
break
|
|
return out
|
|
|
|
|
|
def _iso(moment: datetime) -> str:
|
|
"""UTC, to the second. Every timestamp column is compared as text, so every
|
|
writer has to spell it the same way."""
|
|
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]:
|
|
"""One role's own place texts as stored, in the host's order."""
|
|
return list(dict.fromkeys(r[0] for r in conn.execute(
|
|
"SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid",
|
|
(uid,))))
|
|
|
|
|
|
def _nearest_first(rows: Sequence[RolePlace],
|
|
near: places.Nearness | None) -> list[RolePlace]:
|
|
"""A role's place rows, those of one place text (a copy's, one position)
|
|
kept together, nearest to the agent first (`places.nearest_first`)."""
|
|
groups: dict[tuple[str, int, str], list[RolePlace]] = {}
|
|
for row in rows:
|
|
groups.setdefault((row.url, row.position, row.text), []).append(row)
|
|
entries = list(groups.values())
|
|
order = places.nearest_first(near, [(g[0].text, [r.place_id for r in g]) for g in entries])
|
|
return [row for i in order for row in entries[i]]
|
|
|
|
|
|
def _leads(rows: Sequence[RolePlace]) -> tuple[list[str], str, str]:
|
|
"""`(distinct texts, lead text, lead url)` of places already in order:
|
|
the first place leads, and its url is a copy's link, '' for the role's
|
|
own place."""
|
|
texts = list(dict.fromkeys(t for r in rows if (t := r.text.strip())))
|
|
lead = next((r for r in rows if r.text.strip()), None)
|
|
return texts, lead.text.strip() if lead else "", lead.url if lead else ""
|
|
|
|
|
|
def _like(text: str) -> str:
|
|
"""A `LIKE` pattern for a case-insensitive substring match, with SQLite's
|
|
own wildcards escaped so a search for "50%" or "a_b" stays literal."""
|
|
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
return f"%{escaped}%"
|
|
|
|
|
|
#: Vectors are stored little-endian float32, fixed here so a file written on
|
|
#: one machine reads on another. newsdesk's encoding, written again.
|
|
_DTYPE = np.dtype("<f4")
|
|
|
|
|
|
def pack(vec: np.ndarray) -> bytes:
|
|
return np.asarray(vec, dtype=_DTYPE).tobytes()
|
|
|
|
|
|
def unpack(blob: bytes) -> np.ndarray:
|
|
return np.frombuffer(blob, dtype=_DTYPE)
|
|
|
|
|
|
def _unplaced(gaz: places.Gazetteer) -> str:
|
|
"""A role with no place the gazetteer resolved, over `roles r`. Anywhere
|
|
is no place for this, should a person ever alias it."""
|
|
other = f" AND place_id != {int(gaz.anywhere)}" if gaz.anywhere is not None else ""
|
|
return f"r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL{other})"
|
|
|
|
|
|
#: A remote role whose area is stated (`role_remote`), over `roles r`.
|
|
_HAS_AREA = "(r.remote = 'remote' AND r.uid IN (SELECT uid FROM role_remote))"
|
|
|
|
|
|
def _areas_sql(ids: Iterable[int], table: str) -> tuple[str, list[Any]]:
|
|
"""A remote role whose area holds one of these places, over `table`."""
|
|
return (f"({table}.remote = 'remote' AND {table}.uid IN (SELECT uid FROM role_remote "
|
|
"WHERE place_id IN (SELECT value FROM json_each(?))))", [json.dumps(sorted(ids))])
|
|
|
|
|
|
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
|
"""One gate as `(empty, match, params)`: SQL that is true when the role's
|
|
column is empty, and SQL that is true when it matches, over `roles r`.
|
|
`params` belong to `match`; `empty` takes none.
|
|
|
|
`country` and `place` match over the gazetteer, as the feed's Place
|
|
filter does (`_places_sql`): a country gate is met by a role whose
|
|
`country` is one of its codes or that has a place in one of them, so
|
|
`DK` is met by `Copenhagen; London`. Its column is empty only when the
|
|
role has neither. A `require` gate on country reads a remote role whose
|
|
area is stated through the area alone, as the remote chips do
|
|
(`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)"
|
|
meet Europe, "Remote US" does not, and neither is empty. A place gate's
|
|
value is a place token, bare text a `place:`.
|
|
|
|
`remote_from` takes place tokens without their `remote:` (`country:DK`,
|
|
`anywhere`) and matches as the remote chips do. It is empty for a role
|
|
whose remote is unknown, and for a remote role with no area stated; a
|
|
hybrid or onsite role fails it."""
|
|
column = f"r.{gate.field}"
|
|
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
|
marks = ", ".join("?" for _ in values) or "NULL"
|
|
if gate.field == "salary_min":
|
|
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
|
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
|
if gate.field == "remote_from":
|
|
tokens = [f"remote:{str(v).strip()}" for v in gate.values if str(v).strip()]
|
|
match, params = _places_sql(tokens, gaz, table="r")
|
|
return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params
|
|
if gate.field == "country":
|
|
tokens = [f"country:{v}" for v in values]
|
|
match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values])
|
|
empty = f"(r.country = '' AND {_unplaced(gaz)})"
|
|
if gate.mode != "require":
|
|
return empty, match, params
|
|
named = {p for t in tokens for p in gaz.lookup(t) or ()}
|
|
area, more = _areas_sql(gaz.overlap(named) if named else (), "r")
|
|
return (f"({empty} AND NOT {_HAS_AREA})",
|
|
f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params])
|
|
if gate.field == "place":
|
|
match, params = _places_sql(values, gaz, table="r")
|
|
return "r.place = ''", match, params
|
|
if gate.field == "title":
|
|
# Whole words of the folded title: " account executive " inside
|
|
# " enterprise account executive uk ". A fold drops `%` and `_`, so a
|
|
# value cannot smuggle a LIKE wildcard in.
|
|
words = [f"% {title_key(v)} %" for v in values if title_key(v)]
|
|
match = " OR ".join("(' ' || r.title_key || ' ') LIKE ?" for _ in words) or "0"
|
|
return "r.title_key = ''", f"({match})", words
|
|
if gate.field == "company":
|
|
keys = [company_key(v) for v in values]
|
|
return ("r.company_key = ''",
|
|
f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys)
|
|
if gate.field == "languages":
|
|
return ("r.languages IN ('', '[]')",
|
|
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
|
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
|
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
|
|
|
|
|
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
|
|
codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]:
|
|
"""Any of the place tokens over `table`, as a join over the gazetteer: a
|
|
role with a place under one of the places the tokens name (`country:DK`
|
|
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
|
|
Malmö by any spelling), or whose `country` is one of their countries (a
|
|
role whose board gave a country and no place). `codes` are more values of
|
|
`country` to accept. A `place:` token the gazetteer does not know is
|
|
matched on whole words of the role's folded line (`places.match_sql`),
|
|
so a town nobody has entered yet is still found.
|
|
|
|
A `remote:` token is a remote role whose area overlaps its places
|
|
(`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere.
|
|
`remote:any` is every remote role that meets or leaves empty each of
|
|
`scope`, the viewing agent's `require` gates on country; with none,
|
|
every remote role."""
|
|
ids: set[int] = set()
|
|
areas: set[int] = set()
|
|
found = {c for c in codes if c}
|
|
clauses: list[str] = []
|
|
params: list[Any] = []
|
|
remote: list[tuple[str, list[Any]]] = []
|
|
for token in tokens:
|
|
tok = places.parse(token)
|
|
if tok.kind == "remote":
|
|
if tok.value == "any":
|
|
remote.append(_in_scope(scope, gaz, table))
|
|
elif named_area := gaz.lookup(token):
|
|
areas |= named_area if tok.value == "anywhere" else gaz.overlap(named_area)
|
|
continue
|
|
named = gaz.lookup(token)
|
|
if named is None:
|
|
sql, values = places.match_sql(token, place=f"{table}.place_fold",
|
|
country=f"{table}.country")
|
|
clauses.append(sql)
|
|
params += values
|
|
continue
|
|
under = gaz.under(named)
|
|
ids |= under
|
|
found |= gaz.codes(under)
|
|
if ids:
|
|
clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places "
|
|
"WHERE place_id IN (SELECT value FROM json_each(?)))")
|
|
params.insert(0, json.dumps(sorted(ids)))
|
|
if found:
|
|
clauses.insert(1 if ids else 0,
|
|
f"{table}.country IN (SELECT value FROM json_each(?))")
|
|
params.insert(1 if ids else 0, json.dumps(sorted(found)))
|
|
if areas:
|
|
remote.append(_areas_sql(areas, table))
|
|
for sql, values in remote:
|
|
clauses.append(sql)
|
|
params += values
|
|
return f"({' OR '.join(clauses) or '0'})", params
|
|
|
|
|
|
def _in_scope(scope: Sequence[Gate], gaz: places.Gazetteer, table: str) -> tuple[str, list[Any]]:
|
|
"""`remote:any` over `table`: a remote role that meets or leaves empty
|
|
each gate of `scope` (`_gate_sql`), so a remote role with no area and no
|
|
place is in every scope."""
|
|
if not scope:
|
|
return f"{table}.remote = 'remote'", []
|
|
parts: list[str] = []
|
|
params: list[Any] = []
|
|
for gate in scope:
|
|
empty, match, values = _gate_sql(gate, gaz)
|
|
parts.append(f"({empty} OR {match})")
|
|
params += values
|
|
return (f"{table}.uid IN (SELECT r.uid FROM roles r WHERE r.remote = 'remote' "
|
|
f"AND {' AND '.join(parts)})", params)
|
|
|
|
|
|
def _known_places(tokens: Sequence[str]) -> None:
|
|
"""ValueError for a token that names nothing, which would otherwise match
|
|
no role without saying why."""
|
|
for token in tokens:
|
|
if not places.known(token):
|
|
raise ValueError(f"{token!r} is not a place this app knows")
|
|
|
|
|
|
def _text_rank(description: str, teaser: str) -> int:
|
|
"""What a role is embedded from, as a rank: 2 its description, 1 its
|
|
teaser, 0 its title alone."""
|
|
return 2 if description else 1 if teaser else 0
|
|
|
|
|
|
def _int_or_none(value: Any) -> int | None:
|
|
if value is None or value == "" or isinstance(value, bool):
|
|
return None
|
|
try:
|
|
return int(float(value))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
#: The host `app.dupes.Copy.host` and `RoleCopy.host` read: the provider of
|
|
#: a search's row, else the ATS a board runs on, as `posted_host` reads it.
|
|
_HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')"
|
|
|
|
|
|
#: A copy that is not one place of its kept role: the copies a role lists
|
|
#: under "Also posted" and counts as "x N". `{kept}` is the kept role's uid
|
|
#: as SQL, and a place copy is the one whose link the kept role holds.
|
|
_NOT_A_PLACE = ("original_url NOT IN (SELECT url FROM role_places "
|
|
"WHERE uid = {kept} AND url != '')")
|
|
|
|
|
|
class DupeCounts(NamedTuple):
|
|
"""What `rebuild_dupes` left and did: kept roles with at least one copy,
|
|
open roles that are a copy, and rows whose `dup_of` it rewrote."""
|
|
|
|
groups: int
|
|
copies: int
|
|
changed: int
|
|
|
|
|
|
def _assign_only(rows: Iterable[dupes.Copy]) -> dupes.Folded:
|
|
return dupes.Folded(dupes.assign(rows), frozenset())
|
|
|
|
|
|
def _sync_copy_places(conn: sqlite3.Connection, placed: dict[str, str | None]) -> None:
|
|
"""Make the `role_places` rows with a `url` those of the copies `placed`
|
|
(`copy uid -> kept uid`): each own place of a copy, on the kept role,
|
|
with the copy's link. Written only when they differ from what is stored,
|
|
so an unchanged database is not touched. A copy that closed, or a group
|
|
that dissolved, is no longer in `placed`, and its rows go."""
|
|
want: set[tuple[str, int, str, str, int | None, str]] = set()
|
|
if placed:
|
|
marks = ",".join("?" * len(placed))
|
|
for uid, pos, text, fold, place_id, url in conn.execute(
|
|
f"""SELECT rp.uid, rp.position, rp.text, rp.fold, rp.place_id, r.original_url
|
|
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
|
WHERE rp.url = '' AND rp.uid IN ({marks})""", list(placed)):
|
|
want.add((str(placed[uid]), pos, text, fold, place_id, url))
|
|
have = set(conn.execute(
|
|
"SELECT uid, position, text, fold, place_id, url FROM role_places WHERE url != ''"))
|
|
if want != have:
|
|
conn.execute("DELETE FROM role_places WHERE url != ''")
|
|
conn.executemany(
|
|
"INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
|
"VALUES (?, ?, ?, ?, ?, ?)", sorted(want, key=lambda r: (r[0], r[5], r[1], r[2])))
|
|
|
|
|
|
def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
|
"""Recompute `roles.dup_of` from scratch with `app.dupes.fold`, inside
|
|
the caller's transaction: `Store.rebuild_dupes` and migration 18.
|
|
|
|
Only the open roles of a `(company_key, title_key)` with more than one
|
|
open role can be a copy, a few thousand of the ~34k, so only those are
|
|
read. A row whose `dup_of` differs from the rule's answer is rewritten,
|
|
and a `dup_of` left on a closed role, or on one no longer in such a pair,
|
|
is cleared: a copy never outlives its posting, and a closed kept role
|
|
hands its group to the next copy here.
|
|
|
|
Where a group changed (a member's `dup_of` was rewritten), its kept role
|
|
takes over what was done to the posting under another uid. The sources
|
|
are the group's members and the roles they were copies of before, so a
|
|
closed kept role still hands over. The kept role takes the earliest
|
|
`read_at` of them when it has none, and each agent's latest vote among
|
|
them moves to it unless that agent already voted on it; a vote that
|
|
cannot move stays where it is. An unchanged group is not touched, so
|
|
marking a kept role unread again sticks. Applications keep their
|
|
`role_uid`, and scores of a copy go at the agent's next `score`.
|
|
|
|
A copy the host published for another place (`dupes.Folded.placed`)
|
|
leaves its places on the kept role as `role_places` rows with the copy's
|
|
link, see `_sync_copy_places`."""
|
|
# Migration 18 runs this on a file that has no `teaser` and no
|
|
# `role_places` yet: there it groups as before and folds no places.
|
|
current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)"))
|
|
text = "COALESCE(NULLIF(description, ''), teaser)" if current else "description"
|
|
rows = conn.execute(
|
|
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
|
description != '' AS described, posted_at,
|
|
SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url
|
|
FROM roles
|
|
WHERE closed_at IS NULL AND company_key != ''
|
|
AND (company_key, title_key) IN (
|
|
SELECT company_key, title_key FROM roles
|
|
WHERE closed_at IS NULL AND company_key != ''
|
|
GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall()
|
|
made = dupes.fold if current else _assign_only
|
|
folded = made(
|
|
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
|
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None,
|
|
text=r[8] or "", url=r[9] or "")
|
|
for r in rows)
|
|
new = folded.dup_of
|
|
old: dict[str, str | None] = dict.fromkeys(new)
|
|
old.update((r[0], r[1]) for r in conn.execute(
|
|
"SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL"))
|
|
updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was]
|
|
conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates)
|
|
|
|
if current:
|
|
_sync_copy_places(conn, {uid: new[uid] for uid in folded.placed})
|
|
changed = {uid for _, uid in updates}
|
|
members: dict[str, set[str]] = {}
|
|
for uid, dup in new.items():
|
|
members.setdefault(dup or uid, set()).add(uid)
|
|
groups = {kept: group for kept, group in members.items() if len(group) > 1}
|
|
for kept, group in groups.items():
|
|
if not group & changed:
|
|
continue
|
|
sources = (group | {old.get(uid) or uid for uid in group}) - {kept}
|
|
_carry(conn, kept, sources)
|
|
return DupeCounts(groups=len(groups), copies=sum(dup is not None for dup in new.values()),
|
|
changed=len(updates))
|
|
|
|
|
|
def _carry(conn: sqlite3.Connection, kept: str, sources: set[str]) -> None:
|
|
"""`rebuild_dupes`' hand-over to a group's kept role: the earliest
|
|
`read_at` of `sources` if it has none, and each agent's latest vote among
|
|
them unless it has one of its own."""
|
|
wanted = json.dumps(sorted(sources))
|
|
conn.execute(
|
|
"""UPDATE roles SET read_at = (SELECT MIN(read_at) FROM roles
|
|
WHERE uid IN (SELECT value FROM json_each(?)))
|
|
WHERE uid = ? AND read_at IS NULL""", (wanted, kept))
|
|
voted = {r[0] for r in conn.execute("SELECT agent_id FROM role_votes WHERE uid = ?", (kept,))}
|
|
moves: dict[int, str] = {}
|
|
for agent_id, uid in conn.execute(
|
|
"""SELECT agent_id, uid FROM role_votes WHERE uid IN (SELECT value FROM json_each(?))
|
|
ORDER BY voted_at, uid""", (wanted,)):
|
|
if agent_id not in voted:
|
|
moves[agent_id] = uid # the latest vote wins, by the order above
|
|
conn.executemany("UPDATE role_votes SET uid = ? WHERE agent_id = ? AND uid = ?",
|
|
[(kept, agent_id, uid) for agent_id, uid in moves.items()])
|
|
|
|
|
|
class _StateMap(MutableMapping[str, str]):
|
|
"""`Store.state`: the key/value table, read and written one key at a time."""
|
|
|
|
def __init__(self, store: Store) -> None:
|
|
self._store = store
|
|
|
|
def __getitem__(self, key: str) -> str:
|
|
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
|
|
if row is None:
|
|
raise KeyError(key)
|
|
return str(row["value"])
|
|
|
|
def __setitem__(self, key: str, value: str) -> None:
|
|
self._store._write(
|
|
"""INSERT INTO state (key, value) VALUES (?,?)
|
|
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
|
|
(key, value))
|
|
|
|
def __delitem__(self, key: str) -> None:
|
|
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
|
|
raise KeyError(key)
|
|
|
|
def __iter__(self) -> Iterator[str]:
|
|
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
|
|
|
|
def __len__(self) -> int:
|
|
row = self._store._row("SELECT COUNT(*) AS n FROM state")
|
|
return int(row["n"]) if row else 0
|
|
|
|
|
|
class Store:
|
|
def __init__(self, path: str | Path) -> None:
|
|
self.path = Path(path).expanduser()
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
#: What "now" is. Replaced by tests that need two writes a second apart.
|
|
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
|
#: The gazetteer as last read, with the `gazetteer.VERSION` it was read at.
|
|
self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None
|
|
with self.transaction() as conn:
|
|
# Persistent: recorded in the file, inherited by every connection.
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
#: Migration versions this open applied; empty when already current.
|
|
self.migrations_applied: list[int] = schema.initialise(conn)
|
|
|
|
# ------------------------------------------------------------ connections
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
"""A fresh connection. `foreign_keys` is off by default per connection;
|
|
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
|
`synchronous=NORMAL` is the standard WAL companion. `place_match` is
|
|
registered on every one, since the place filter and the place gate
|
|
call it for a town the gazetteer does not know (`_places_sql`)."""
|
|
conn = sqlite3.connect(self.path)
|
|
conn.row_factory = sqlite3.Row
|
|
places.register(conn)
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
conn.execute("PRAGMA busy_timeout=30000")
|
|
conn.execute("PRAGMA synchronous=NORMAL")
|
|
return conn
|
|
|
|
@contextmanager
|
|
def transaction(self) -> Iterator[sqlite3.Connection]:
|
|
"""One connection, one transaction, closed on the way out."""
|
|
conn = self._connect()
|
|
try:
|
|
with conn:
|
|
yield conn
|
|
finally:
|
|
conn.close()
|
|
|
|
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
|
with closing(self._connect()) as conn:
|
|
return conn.execute(sql, params).fetchall()
|
|
|
|
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
|
with closing(self._connect()) as conn:
|
|
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
|
|
return row
|
|
|
|
def _write(self, sql: str, params: Any = ()) -> int:
|
|
"""One statement in its own transaction. Returns the rowcount."""
|
|
with self.transaction() as conn:
|
|
return conn.execute(sql, params).rowcount
|
|
|
|
def now(self) -> str:
|
|
return _iso(self.clock())
|
|
|
|
# ------------------------------------------------------------- gazetteer
|
|
|
|
def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer:
|
|
"""The gazetteer, read again only when its tables have grown, which
|
|
another process may have done: one cheap query a call."""
|
|
version = tuple(conn.execute(gazetteer.VERSION).fetchone())
|
|
if self._gaz is None or self._gaz[0] != version:
|
|
self._gaz = (version, gazetteer.load(conn))
|
|
return self._gaz[1]
|
|
|
|
def gazetteer(self) -> places.Gazetteer:
|
|
with closing(self._connect()) as conn:
|
|
return self._gazetteer(conn)
|
|
|
|
def role_places(self, uid: str) -> list[RolePlace]:
|
|
"""One role's places in the host's order, a copy's after the role's own."""
|
|
return [RolePlace.from_row(r) for r in self._rows(
|
|
"""SELECT rp.position, rp.text, rp.place_id, rp.url,
|
|
COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind
|
|
FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id
|
|
WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""",
|
|
(uid,))]
|
|
|
|
def unresolved_places(self, open_only: bool = True,
|
|
limit: int | None = None) -> list[tuple[str, str, int]]:
|
|
"""`(fold, text, roles)` for every place text no place of the
|
|
gazetteer resolves, most roles first: the list a person works through
|
|
with `add_place_alias` and `add_place`. `text` is one spelling of the
|
|
fold. A text that names no place ("Remote", `places.placeless`) is
|
|
left out; with `open_only`, so are closed roles and copies."""
|
|
where = "rp.place_id IS NULL"
|
|
if open_only:
|
|
where += " AND r.closed_at IS NULL AND r.dup_of IS NULL"
|
|
rows = self._rows(
|
|
f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n
|
|
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
|
WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""")
|
|
found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows
|
|
if not places.placeless(r["fold"])]
|
|
return found[:limit] if limit is not None else found
|
|
|
|
def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int:
|
|
"""A place by id, or by a token that names exactly one place."""
|
|
if isinstance(place, int):
|
|
if place not in gaz.places:
|
|
raise ValueError(f"there is no place {place}")
|
|
return place
|
|
named = gaz.lookup(place) or frozenset()
|
|
if len(named) != 1:
|
|
raise ValueError(f"{place!r} names {len(named)} places, not one")
|
|
return next(iter(named))
|
|
|
|
def add_place_alias(self, text: str, place: int | str) -> int:
|
|
"""Write `text` as a spelling of `place` (an id, or a token such as
|
|
`place:copenhagen` or `country:DK` naming one place), and resolve again
|
|
every role with a place text that holds it. Returns how many roles now
|
|
resolve differently."""
|
|
folded = places.fold(text)
|
|
if not folded:
|
|
raise ValueError("an alias needs a letter or a digit")
|
|
with self.transaction() as conn:
|
|
place_id = self._place_id(self._gazetteer(conn), place)
|
|
conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
|
(folded, place_id))
|
|
return gazetteer.relink(conn, self._gazetteer(conn), folded)
|
|
|
|
def add_place(self, name: str, kind: str, parent: int | str,
|
|
aliases: Sequence[str] = ()) -> int:
|
|
"""Add a city or a region the gazetteer lacks, in `parent` (an id or a
|
|
token), written as `name` and each of `aliases`, and resolve again the
|
|
roles with a place text that holds one of them. Returns its id."""
|
|
if kind not in ("city", "region"):
|
|
raise ValueError(f"a place added by hand is a city or a region, not {kind!r}")
|
|
folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f))
|
|
if not folds:
|
|
raise ValueError("a place needs a name")
|
|
with self.transaction() as conn:
|
|
parent_id = self._place_id(self._gazetteer(conn), parent)
|
|
if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?",
|
|
(kind, folds[0])).fetchone():
|
|
raise ValueError(f"there is already a {kind} {name!r}")
|
|
place_id = int(conn.execute(
|
|
"INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)",
|
|
(kind, folds[0], name.strip(), parent_id)).lastrowid or 0)
|
|
conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)",
|
|
[(f, place_id) for f in folds])
|
|
gaz = self._gazetteer(conn)
|
|
for folded in folds:
|
|
gazetteer.relink(conn, gaz, folded)
|
|
return place_id
|
|
|
|
def unlinked_places(self, limit: int | None = None) -> list[dict[str, Any]]:
|
|
"""`unresolved_places` with what places each text: `countries`, the
|
|
`[code, roles]` prep gave its roles ('' for none), and `companies`,
|
|
the three that write it most, both most roles first. What the Sources
|
|
page's Unmapped section and `app prep unlinked` list."""
|
|
found = self.unresolved_places(limit=limit)
|
|
countries: dict[str, Counter[str]] = {fold: Counter() for fold, _, _ in found}
|
|
companies: dict[str, Counter[str]] = {fold: Counter() for fold, _, _ in found}
|
|
for r in self._rows(
|
|
"""SELECT rp.fold, r.country, r.company, COUNT(DISTINCT r.uid) AS n
|
|
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
|
WHERE rp.place_id IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL
|
|
GROUP BY rp.fold, r.country, r.company"""):
|
|
if (fold := str(r["fold"])) in countries:
|
|
countries[fold][str(r["country"])] += int(r["n"])
|
|
companies[fold][str(r["company"])] += int(r["n"])
|
|
return [{"fold": fold, "text": text, "roles": n,
|
|
"countries": [list(c) for c in countries[fold].most_common()],
|
|
"companies": [c for c, _ in companies[fold].most_common(3) if c]}
|
|
for fold, text, n in found]
|
|
|
|
def seed_places(self) -> int:
|
|
"""Add what `places.seed` describes and the database lacks, a place
|
|
or a spelling written into the code since the file was made. Prep
|
|
calls it when the code changed and resolves the open roles again.
|
|
Returns how many places were added."""
|
|
with self.transaction() as conn:
|
|
return gazetteer.seed(conn)
|
|
|
|
def relink_places(self) -> int:
|
|
"""Resolve every role's own places again, after the gazetteer was
|
|
seeded anew. Returns how many roles now resolve differently."""
|
|
with self.transaction() as conn:
|
|
return gazetteer.relink(conn, self._gazetteer(conn))
|
|
|
|
# ---------------------------------------------------------------- roles
|
|
|
|
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
|
|
"""Store one posting. Returns True when it was new.
|
|
|
|
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
|
every call, and a role seen again is open again. The raw columns come
|
|
from `item.raw` under their own names, with `company` falling back to
|
|
the subscription's; an empty value never overwrites a filled column.
|
|
|
|
Nothing is mapped here: that is `app.prep`, which runs after every
|
|
pull. The host's words go to `seniority_host`, `employment_host`,
|
|
`remote_host` and `country_host` (`host_word`), `company_key` is
|
|
written unresolved on a new role only, and the mapped columns
|
|
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
|
|
`remote`, `country`) are left as prep last wrote them. `prep_due`
|
|
is 1 on a new role and grows by one when the title, the company, a
|
|
host word, the host's deadline or the place texts changed or the
|
|
role reopens, which is what the next prep reads. `deadline_host` is
|
|
`raw["applicationDeadline"]` as `YYYY-MM-DD` (`iso_day`), '' when
|
|
the pull states none, and prep composes `deadline` from it.
|
|
|
|
`posted_at` (see `app.models.posted_at`) is set from this pull's own
|
|
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
|
|
keeps the earlier of that and what is already stored, so an edit on
|
|
the board — which can only ever move `published_at` later, never
|
|
earlier — cannot move `posted_at` later either.
|
|
|
|
`function` is the one gate column no board actually sends: the `ats`
|
|
plugin's own keyword guess rides in `raw` under that name because
|
|
`raw` is the only channel a plugin has, but a guess is not first-party
|
|
the way a board's own structured field is (docs/decisions.md, "a
|
|
guess is not a board field"). So once a role has been through
|
|
`extract` (`extracted_at` is set), this upsert leaves `function`
|
|
alone for good, whatever the plugin's raw guess now says — see
|
|
`set_extracted` for the other half of the rule.
|
|
|
|
The role's places (`role_places`) are the texts of `raw["places"]`,
|
|
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
|
when they differ from the stored ones; prep resolves them. Texts that
|
|
did not change keep their resolved rows, so the feed's Place filter
|
|
stays right between a pull and its prep. `place` is the line shown,
|
|
the texts joined when `raw` has none. Like `place`, a pull with no
|
|
place leaves them as they are. `place_fold` is written with `place`,
|
|
under the same keep-filled rule. `teaser` is kept-filled too.
|
|
|
|
A role is embedded from its description, else its teaser, else its
|
|
title (`app.jobs.embed.document`). When a pull brings a better one of
|
|
these than the role had, its vector is dropped so the next `embed`
|
|
reads it.
|
|
"""
|
|
if not item.source:
|
|
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
|
raw = item.raw or {}
|
|
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
|
place_texts = gazetteer.texts(raw)
|
|
place = str(raw.get("place") or "") or "; ".join(place_texts)
|
|
stamp = self.now()
|
|
published = _iso(item.published_at)
|
|
row: dict[str, Any] = {
|
|
"uid": item.uid,
|
|
"source": item.source,
|
|
"source_id": item.source_id,
|
|
"subscription_id": subscription.id if subscription else None,
|
|
"company_key": company_key(company),
|
|
"company": company,
|
|
"title": item.title,
|
|
"title_key": title_key(item.title),
|
|
"place_key": place_key(place),
|
|
"description": item.body,
|
|
"feed_text": item.excerpt,
|
|
"original_url": item.location,
|
|
"published_at": published,
|
|
"posted_at": posted_at(published, stamp, posted_host(raw)),
|
|
"first_seen": stamp,
|
|
"last_seen": stamp,
|
|
"raw": json.dumps(raw, default=str),
|
|
"created_at": stamp,
|
|
}
|
|
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
|
row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS}
|
|
row["place"] = place
|
|
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
|
row["function"] = str(raw.get("function") or "")
|
|
row["teaser"] = str(raw.get("teaser") or "")
|
|
row["place_fold"] = places.fold(row["place"])
|
|
# A board's own pay line, one per pull like title or published_at —
|
|
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
|
# must be able to clear it, not leave a stale line behind.
|
|
row["salary_text"] = str(raw.get("salary_text") or "")
|
|
row["deadline_host"] = iso_day(raw.get("applicationDeadline"))
|
|
row["prep_due"] = 1
|
|
|
|
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
|
|
"teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS)
|
|
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
|
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
|
for c in (*_RAW_INT, "subscription_id")]
|
|
updates += [f"{c}=excluded.{c}"
|
|
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
|
"salary_text", "seniority_host", "deadline_host")]
|
|
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
|
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
|
# pull's classifier guess overwrite what extract decided, which is
|
|
# the flip the "board wins" rule exists to prevent for an actual
|
|
# board field — see the docstring above. Once extracted_at is set,
|
|
# function is extract's to own; before that, it is the guess, same
|
|
# as any other raw field.
|
|
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
|
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
|
"ELSE roles.function END")
|
|
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
|
"THEN excluded.place_fold ELSE roles.place_fold END")
|
|
# One more than before when what prep maps from changed, or the role
|
|
# reopens, so a prep that read the old values leaves it due
|
|
# (`Store.prepped`). `roles.` is the stored row here, `excluded.`
|
|
# what this pull brought.
|
|
changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN]
|
|
changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})"
|
|
for c in ("company", *_KEPT_HOST_WORDS)]
|
|
updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} "
|
|
"THEN roles.prep_due + 1 ELSE roles.prep_due END")
|
|
updates.append("closed_at=NULL")
|
|
columns = ", ".join(row)
|
|
marks = ", ".join(f":{c}" for c in row)
|
|
with self.transaction() as conn:
|
|
before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?",
|
|
(row["uid"],)).fetchone()
|
|
conn.execute(
|
|
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
|
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
|
if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts):
|
|
gazetteer.write_texts(conn, row["uid"], place_texts)
|
|
if before is not None:
|
|
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
|
(row["uid"],))
|
|
# `embed` encodes a role once, from the best text it has. When a
|
|
# later pull brings a better one, as a host that states the text
|
|
# on the role's own page does, the vector goes, so the next
|
|
# `embed` reads it.
|
|
if before is not None and _text_rank(row["description"] or before["description"],
|
|
row["teaser"] or before["teaser"]) \
|
|
> _text_rank(before["description"], before["teaser"]):
|
|
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
|
|
return before is None
|
|
|
|
def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None:
|
|
"""One role with its tags, and with `vote`, `voted_at`, `score`,
|
|
`gated_reason`, `prompt` and `cv_fit` as `agent_id` judged and scored
|
|
it, and its `judgement`, `cv_score` (under the agent's weights as they
|
|
are now) and language flag for that agent; None
|
|
(unjudged, unscored) without an agent. The score is moved by the vote
|
|
as `roles` serves it."""
|
|
row = self._row(
|
|
f"SELECT roles.*, v.vote, v.voted_at, {_VOTED_SCORE} AS score, "
|
|
"COALESCE(s.gated_reason, '') AS gated_reason, COALESCE(s.prompt, '') AS prompt, "
|
|
f"s.cv_fit FROM roles {_WITH_VOTES} "
|
|
"LEFT JOIN scores s ON s.uid = roles.uid AND s.agent_id = ? WHERE roles.uid=?",
|
|
(agent_id, agent_id, uid))
|
|
if row is None:
|
|
return None
|
|
role = ScoredRole.from_row(row)
|
|
role.tags = self._tags_of(uid)
|
|
role.copies = self._copies_of(uid)
|
|
role.places = _nearest_first(self.role_places(uid), self._nearness(agent_id))
|
|
_, role.lead_place, role.lead_url = _leads(role.places)
|
|
role.lead_place = role.lead_place or role.place
|
|
role.remote_where = self._remote_where([uid]).get(uid, "")
|
|
role.judgement = self.judgement(agent_id, uid)
|
|
role.cv_score = role.judgement.cv_score if role.judgement else None
|
|
role.lang_flag, role.lang_note = language_flag(role.languages_required,
|
|
self._languages_of(agent_id))
|
|
return role
|
|
|
|
def _copies_of(self, uid: str) -> list[RoleCopy]:
|
|
"""The open copies of one kept role, oldest first; empty for a copy
|
|
or a role without any. See `Role.copies`."""
|
|
return [RoleCopy.from_row(r) for r in self._rows(
|
|
f"""SELECT uid, {_HOST} AS host, place, original_url, posting_url, posted_at FROM roles
|
|
WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")}
|
|
ORDER BY posted_at, uid""", (uid, uid))]
|
|
|
|
def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]:
|
|
"""`uid -> open copies` for a page of kept roles, one grouped query
|
|
over `idx_roles_dup_of`. A role without copies is absent. The index
|
|
is named because the planner otherwise takes `idx_roles_closed` for
|
|
`closed_at IS NULL` and reads every open role, 19 ms against 0.1 on
|
|
the live database."""
|
|
if not uids:
|
|
return {}
|
|
return {str(r["dup_of"]): int(r["n"]) for r in self._rows(
|
|
"SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of "
|
|
"WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL "
|
|
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
|
|
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
|
|
|
def _remote_where(self, uids: Sequence[str]) -> dict[str, str]:
|
|
"""`uid -> the area as a label` ("Europe", "United States, Canada",
|
|
"anywhere", `Gazetteer.area_label`) for the remote roles of `uids`
|
|
with an area stated; a role without one is absent. See
|
|
`Role.remote_where`."""
|
|
if not uids:
|
|
return {}
|
|
found: dict[str, list[int]] = {}
|
|
for r in self._rows(
|
|
"""SELECT rr.uid, rr.place_id FROM role_remote rr
|
|
JOIN roles ON roles.uid = rr.uid AND roles.remote = 'remote'
|
|
WHERE rr.uid IN (SELECT value FROM json_each(?))
|
|
ORDER BY rr.uid, rr.rowid""", (json.dumps(list(uids)),)):
|
|
found.setdefault(str(r["uid"]), []).append(int(r["place_id"]))
|
|
if not found:
|
|
return {}
|
|
gaz = self.gazetteer()
|
|
return {uid: gaz.area_label(ids) for uid, ids in found.items()}
|
|
|
|
def _nearness(self, agent_id: int | None) -> places.Nearness | None:
|
|
"""How near a place is to the agent a page is viewed as, from its
|
|
gates; None without an agent, which keeps the host's order."""
|
|
found = self.agent(agent_id) if agent_id is not None else None
|
|
return places.Nearness(self.gazetteer(), found.gates) if found is not None else None
|
|
|
|
def _place_leads(self, uids: Sequence[str],
|
|
near: places.Nearness | None) -> dict[str, list[RolePlace]]:
|
|
"""`uid -> its places nearest first` (`_nearest_first`) for a page
|
|
of roles; a role with no place is absent. One query over the page's
|
|
uids, so only the roles shown are ordered."""
|
|
if not uids:
|
|
return {}
|
|
found: dict[str, list[RolePlace]] = {}
|
|
for r in self._rows(
|
|
"""SELECT uid, position, text, place_id, url FROM role_places
|
|
WHERE uid IN (SELECT value FROM json_each(?))
|
|
ORDER BY uid, url != '', url, position, rowid""",
|
|
(json.dumps(list(uids)),)):
|
|
found.setdefault(str(r["uid"]), []).append(RolePlace(
|
|
position=r["position"], text=r["text"], place_id=r["place_id"], url=r["url"]))
|
|
return {uid: _nearest_first(rows, near) for uid, rows in found.items()}
|
|
|
|
def rebuild_dupes(self) -> DupeCounts:
|
|
"""Recompute which open roles are copies of one posting, in one
|
|
transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE`
|
|
because it reads before it writes: a deferred transaction that a
|
|
pull's upsert wrote under in between could not take the write lock
|
|
and would fail rather than wait."""
|
|
with self.transaction() as conn:
|
|
conn.execute("BEGIN IMMEDIATE")
|
|
return rebuild_dupes(conn)
|
|
|
|
def set_read(self, uid: str, read: bool = True) -> Role:
|
|
"""Mark a role read, or unread again. Stamps `read_at` only the first
|
|
time — a role already read keeps the moment it was first opened —
|
|
and clears it outright when `read` is False. Raises KeyError for an
|
|
unknown uid."""
|
|
if read:
|
|
changed = self._write(
|
|
"UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?",
|
|
(self.now(), uid))
|
|
else:
|
|
changed = self._write("UPDATE roles SET read_at = NULL WHERE uid = ?", (uid,))
|
|
if not changed:
|
|
raise KeyError(uid)
|
|
role = self.role(uid)
|
|
assert role is not None
|
|
return role
|
|
|
|
def set_vote(self, agent_id: int, uid: str, value: int | None) -> ScoredRole:
|
|
"""One agent's judgement of one role: 1 like, 0 neither way, -1 pass,
|
|
None unjudged again, which deletes the row. A vote never hides a
|
|
role; it moves it in that agent's ranking (see `roles`) and shapes its
|
|
taste direction (see `jobs/score.py`). Voting is reading, so a vote
|
|
stamps the role's `read_at` too, the same once-only way `set_read`
|
|
does; clearing a vote is not un-reading, and leaves it alone. Returns
|
|
the role with this agent's vote. Raises ValueError for another value
|
|
and KeyError for an unknown uid or agent."""
|
|
if isinstance(value, bool) or value not in VOTES:
|
|
raise ValueError(f"a vote is -1, 0 or 1, not {value!r}")
|
|
with self.transaction() as conn:
|
|
self._known(conn, "roles", "uid", uid)
|
|
self._known(conn, "agents", "id", agent_id)
|
|
if value is None:
|
|
conn.execute("DELETE FROM role_votes WHERE agent_id=? AND uid=?",
|
|
(agent_id, uid))
|
|
else:
|
|
now = self.now()
|
|
conn.execute(
|
|
"""INSERT INTO role_votes (agent_id, uid, vote, voted_at) VALUES (?,?,?,?)
|
|
ON CONFLICT(agent_id, uid) DO UPDATE
|
|
SET vote=excluded.vote, voted_at=excluded.voted_at""",
|
|
(agent_id, uid, value, now))
|
|
conn.execute("UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?",
|
|
(now, uid))
|
|
role = self.role(uid, agent_id)
|
|
assert role is not None
|
|
return role
|
|
|
|
def votes(self, agent_id: int) -> dict[str, int]:
|
|
"""`uid -> vote` for one agent, open roles and closed alike: a closed
|
|
role somebody liked still says what they like."""
|
|
return {r["uid"]: int(r["vote"]) for r in self._rows(
|
|
"SELECT uid, vote FROM role_votes WHERE agent_id=?", (agent_id,))}
|
|
|
|
@staticmethod
|
|
def _known(conn: sqlite3.Connection, table: str, column: str, value: Any) -> None:
|
|
"""KeyError unless the row exists: the caller's 404, rather than the
|
|
foreign key's IntegrityError, which does not say which side is unknown."""
|
|
if conn.execute(f"SELECT 1 FROM {table} WHERE {column}=?", (value,)).fetchone() is None:
|
|
raise KeyError(value)
|
|
|
|
def set_tags(self, uid: str, tags: Sequence[str]) -> list[str]:
|
|
"""Replace one role's tags in one transaction. See `_normalise_tags`
|
|
for how each one is cleaned up. Returns what was stored, sorted the
|
|
way `role` and `roles` read it back. Raises KeyError for an unknown
|
|
uid."""
|
|
with self.transaction() as conn:
|
|
if conn.execute("SELECT 1 FROM roles WHERE uid=?", (uid,)).fetchone() is None:
|
|
raise KeyError(uid)
|
|
return self._replace_tags(conn, uid, tags)
|
|
|
|
def _replace_tags(self, conn: sqlite3.Connection, uid: str,
|
|
tags: Sequence[str]) -> list[str]:
|
|
"""`set_tags`'s write, against an already-open transaction so
|
|
`set_summary` can share it."""
|
|
cleaned = _normalise_tags(tags)
|
|
conn.execute("DELETE FROM role_tags WHERE uid=?", (uid,))
|
|
conn.executemany("INSERT INTO role_tags (uid, tag) VALUES (?,?)",
|
|
[(uid, tag) for tag in cleaned])
|
|
return sorted(cleaned)
|
|
|
|
def _tags_of(self, uid: str) -> list[str]:
|
|
return [str(r["tag"]) for r in self._rows(
|
|
"SELECT tag FROM role_tags WHERE uid=? ORDER BY tag", (uid,))]
|
|
|
|
def _tags_by_uid(self, uids: Sequence[str]) -> dict[str, list[str]]:
|
|
"""Every tag of these roles, by uid, sorted — one query for a whole
|
|
page rather than one per role."""
|
|
if not uids:
|
|
return {}
|
|
out: dict[str, list[str]] = {}
|
|
rows = self._rows(
|
|
"SELECT uid, tag FROM role_tags WHERE uid IN (SELECT value FROM json_each(?)) "
|
|
"ORDER BY uid, tag", (json.dumps(list(uids)),))
|
|
for r in rows:
|
|
out.setdefault(r["uid"], []).append(str(r["tag"]))
|
|
return out
|
|
|
|
def roles(self, limit: int = 100, company: str | None = None,
|
|
place: str | None = None, open_only: bool = True,
|
|
q: str | None = None, function: str | None = None,
|
|
remote: str | None = None, days: int | None = None,
|
|
hide_muted: bool = False, include_read: bool = True,
|
|
tags: Sequence[str] = (), tag_mode: str = "any",
|
|
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
|
judged: str | None = None, pay: str | None = None,
|
|
company_keys: Sequence[str] = (), places: Sequence[str] = (),
|
|
seniority: Sequence[str] = (), seniority_unknown: bool = True,
|
|
pay_unknown: bool = False, by_cv: bool = False, closing: bool = False,
|
|
offset: int = 0) -> list[ScoredRole]:
|
|
"""Newest first by `posted_at`. `company` and `place` are
|
|
case-insensitive substrings of the columns a board wrote — not the
|
|
key, so a search does not have to know the exact slug a company folds
|
|
to. `q` is matched word by word against `title` and `company`; a
|
|
role must match every word in one or the other. `function` and
|
|
`remote` match a gate column exactly. `days` keeps only roles posted
|
|
within that many days of now. `pay` (a key of `PAY`) filters on
|
|
`salary_state`; with `pay_unknown` a role that states no pay passes
|
|
it too.
|
|
|
|
`company_keys` keeps those exact companies. `places` keeps a role any
|
|
of the place tokens finds (`app.places`: `country:DK`, `region:skane`,
|
|
`place:malmo`), over the gazetteer (`_places_sql`); a token that names
|
|
nothing is a ValueError. `seniority` keeps the roles at any of those
|
|
levels (`LEVELS`) and, with `seniority_unknown` (the default), those
|
|
with no level, which is most of them. An empty list is no filter.
|
|
|
|
`agent` says whose votes, hidden companies and scores are meant. Each
|
|
row carries that agent's `vote` and `voted_at`; `judged` (a key of
|
|
`JUDGED`: liked, unjudged or passed) filters on them. `hide_muted`
|
|
excludes the companies that agent hid, before `limit` is applied, in
|
|
SQL rather than after the fact: a company that has published a lot,
|
|
all at once — a fresh board added, a re-pull that finally reached
|
|
it — can otherwise fill the whole page with roles that then all turn
|
|
out to be from the one company someone hid, leaving nothing to show
|
|
for a default query that would otherwise have plenty. With no agent
|
|
every vote is None, nothing is hidden and `judged` reads as such.
|
|
|
|
`include_read` set False excludes a role `set_read` or `set_vote` has
|
|
stamped, the same way and for the same reason as `hide_muted`:
|
|
before `limit`, not after. `tags` keeps only roles carrying at least
|
|
one of them (`tag_mode="any"`, the default) or every one
|
|
(`tag_mode="all"`), also before `limit`.
|
|
|
|
With `agent` and `by_score`, ordered by that agent's score instead,
|
|
highest first, carrying `score` and `gated_reason` on each row. Only
|
|
roles it has scored are listed, and a gated one only when `gated` is
|
|
set. The score is the stored one moved by the agent's vote
|
|
(`_VOTED_SCORE`), so a vote shows in the order at once and needs no
|
|
rescore. With `by_score` off the order is newest, with the votes and
|
|
no score.
|
|
|
|
With `by_cv` as well, the roles the agent's `judge` read come first
|
|
by `cv_score`, then the others by how close they are to the CV
|
|
(`cv_fit`), then by score (CV match). Every row
|
|
carries the agent's `cv_score` (None when not judged), computed
|
|
from the three numbers under the agent's weights at read time, and the
|
|
language flag of its languages against the agent's
|
|
(`language_flag`). `closing` keeps the roles whose `deadline` falls
|
|
today or in the next `CLOSING_DAYS` days.
|
|
|
|
`offset` skips that many rows of the same order, for the page that
|
|
loads the next 60 as it is scrolled; `count_roles` is the total.
|
|
Every returned role carries its tags, read with one extra query for
|
|
the whole page rather than one per role."""
|
|
sql, params = self._roles_from(
|
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
|
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
|
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
|
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
|
closing=closing)
|
|
weights = self._weights_of(agent)
|
|
cv = _cv_sql(weights)
|
|
select = f"SELECT roles.*, v.vote, v.voted_at, {cv} AS cv_score"
|
|
order = "roles.posted_at DESC, roles.uid"
|
|
scored = agent is not None and by_score
|
|
if scored:
|
|
select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt, s.cv_fit"
|
|
order = (_by_cv(cv) if by_cv else "") + "score DESC, " + order
|
|
elif open_only:
|
|
# Newest first, open only: idx_roles_posted covers this exactly
|
|
# (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
|
|
# LIMIT rather than sorting every open role first. Left to the
|
|
# planner, sqlite3's bundled build (no STAT4) reads the near-100%
|
|
# selectivity of `closed_at IS NULL` from idx_roles_closed's plain
|
|
# average and picks that instead, sorting the lot — checked with
|
|
# EXPLAIN QUERY PLAN on a copy of the live database.
|
|
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
|
|
found = [ScoredRole.from_row(r) for r in self._rows(
|
|
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
|
|
by_uid = self._tags_by_uid([r.uid for r in found])
|
|
areas = self._remote_where([r.uid for r in found])
|
|
mine = self._languages_of(agent)
|
|
for r in found:
|
|
r.tags = by_uid.get(r.uid, [])
|
|
r.remote_where = areas.get(r.uid, "")
|
|
r.lang_flag, r.lang_note = language_flag(r.languages_required, mine)
|
|
if r.cv_score is not None:
|
|
r.cv_score = round(r.cv_score, 1)
|
|
return found
|
|
|
|
def _weights_of(self, agent_id: int | None) -> Weights | None:
|
|
"""The agent's weights, None with no agent."""
|
|
found = self.agent(agent_id) if agent_id is not None else None
|
|
return found.weights if found else None
|
|
|
|
def _languages_of(self, agent_id: int | None) -> list[CvLanguage]:
|
|
"""The languages the agent's CV states, [] with no agent."""
|
|
found = self.agent(agent_id) if agent_id is not None else None
|
|
return found.cv_languages if found else []
|
|
|
|
def roles_page(self, limit: int = 100, company: str | None = None,
|
|
place: str | None = None, open_only: bool = True,
|
|
q: str | None = None, function: str | None = None,
|
|
remote: str | None = None, days: int | None = None,
|
|
hide_muted: bool = False, include_read: bool = True,
|
|
tags: Sequence[str] = (), tag_mode: str = "any",
|
|
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
|
judged: str | None = None, pay: str | None = None,
|
|
company_keys: Sequence[str] = (), places: Sequence[str] = (),
|
|
seniority: Sequence[str] = (), seniority_unknown: bool = True,
|
|
pay_unknown: bool = False, by_cv: bool = False, closing: bool = False,
|
|
offset: int = 0) -> list[RoleRow]:
|
|
"""What `GET /api/roles` serves: the same rows `roles` would, in the
|
|
same order, with the same filters and paging, but only the fields a
|
|
row draws (see `RoleRow`) — never `description`, `feed_text`, `raw`
|
|
or `article_text`, which a full `roles.*` select would carry whether
|
|
or not a page ever renders them. The reader still gets the whole role
|
|
by uid, see `role`.
|
|
|
|
`lang_flag` and `lang_note` are the role's languages against the
|
|
agent's (`language_flag`), `cv_score` its `judge` total and `cv_fit`
|
|
how close it is to the agent's CV.
|
|
|
|
`excerpt` is `app.models.excerpt` over a `SUBSTR` of `_EXCERPT_SOURCE`
|
|
(`_EXCERPT_RAW` characters, trimmed further in Python), so the source
|
|
column itself never crosses out of SQLite for a list."""
|
|
sql, params = self._roles_from(
|
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
|
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
|
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
|
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
|
closing=closing)
|
|
cv = _cv_sql(self._weights_of(agent))
|
|
select = (
|
|
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
|
|
"roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, "
|
|
"roles.salary_max, roles.currency, roles.salary_text, roles.original_url, "
|
|
"roles.posting_url, roles.posted_at, roles.read_at, roles.deadline, "
|
|
f"roles.languages_required, v.vote, v.voted_at, {cv} AS cv_score, "
|
|
f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt")
|
|
order = "roles.posted_at DESC, roles.uid"
|
|
scored = agent is not None and by_score
|
|
if scored:
|
|
select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt, s.cv_fit"
|
|
order = (_by_cv(cv) if by_cv else "") + "score DESC, " + order
|
|
elif open_only:
|
|
# Same reasoning as `roles`: the planner otherwise sorts every
|
|
# open role before applying LIMIT.
|
|
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
|
|
rows = self._rows(f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?",
|
|
[*params, limit, offset])
|
|
mine = self._languages_of(agent)
|
|
found = []
|
|
for r in rows:
|
|
row = dict(r)
|
|
row["excerpt"] = excerpt(row["excerpt"])
|
|
if row["cv_score"] is not None:
|
|
row["cv_score"] = round(row["cv_score"], 1)
|
|
if row.get("cv_fit") is not None:
|
|
row["cv_fit"] = round(row["cv_fit"], 3)
|
|
row["lang_flag"], row["lang_note"] = language_flag(
|
|
role_languages(row.pop("languages_required")), mine)
|
|
found.append(RoleRow.from_row(row))
|
|
by_uid = self._tags_by_uid([f.uid for f in found])
|
|
copies = self._copy_counts([f.uid for f in found])
|
|
leads = self._place_leads([f.uid for f in found], self._nearness(agent))
|
|
areas = self._remote_where([f.uid for f in found])
|
|
for f in found:
|
|
f.tags = by_uid.get(f.uid, [])
|
|
f.copies = copies.get(f.uid, 0)
|
|
f.place_texts, f.lead_place, f.lead_url = _leads(leads.get(f.uid, []))
|
|
if len(f.place_texts) > 1:
|
|
f.place = dupes.place_line(f.place_texts)
|
|
f.lead_place = f.lead_place or f.place
|
|
f.remote_where = areas.get(f.uid, "")
|
|
return found
|
|
|
|
def count_roles(self, company: str | None = None, place: str | None = None,
|
|
open_only: bool = True, q: str | None = None,
|
|
function: str | None = None, remote: str | None = None,
|
|
days: int | None = None, hide_muted: bool = False,
|
|
include_read: bool = True, tags: Sequence[str] = (),
|
|
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
|
gated: bool = False, judged: str | None = None,
|
|
pay: str | None = None, company_keys: Sequence[str] = (),
|
|
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
|
seniority_unknown: bool = True, pay_unknown: bool = False,
|
|
by_cv: bool = False, closing: bool = False) -> int:
|
|
"""How many roles `roles` would list with the same filters and no limit."""
|
|
sql, params = self._roles_from(
|
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
|
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
|
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
|
seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown,
|
|
closing=closing)
|
|
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
|
|
return int(row["n"]) if row else 0
|
|
|
|
def role_facets(self, company: str | None = None, place: str | None = None,
|
|
open_only: bool = True, q: str | None = None,
|
|
function: str | None = None, remote: str | None = None,
|
|
days: int | None = None, hide_muted: bool = False,
|
|
include_read: bool = True, tags: Sequence[str] = (),
|
|
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
|
gated: bool = False, judged: str | None = None,
|
|
pay: str | None = None, company_keys: Sequence[str] = (),
|
|
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
|
seniority_unknown: bool = True,
|
|
pay_unknown: bool = False, by_cv: bool = False,
|
|
closing: bool = False) -> dict[str, list[tuple[str, int]]]:
|
|
"""How many roles `roles` would list at each seniority level and each
|
|
pay state, each counted with every other filter and its own left
|
|
out, so the rail can say what a click would add. `seniority` is every
|
|
level of `LEVELS` in order and then `""`, no level; `pay` is
|
|
`salary_state`'s 'normalised' and 'stated' (a line with no figure)
|
|
and then `""`, no pay stated. A value no role has counts 0."""
|
|
shared: dict[str, Any] = {
|
|
"company": company, "place": place, "open_only": open_only, "q": q,
|
|
"function": function, "remote": remote, "days": days, "hide_muted": hide_muted,
|
|
"include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent,
|
|
"by_score": by_score, "gated": gated, "judged": judged,
|
|
"company_keys": company_keys, "places": places, "closing": closing}
|
|
by_level, level_params = self._roles_from(
|
|
**shared, pay=pay, pay_unknown=pay_unknown)
|
|
by_pay, pay_params = self._roles_from(
|
|
**shared, seniority=seniority, seniority_unknown=seniority_unknown)
|
|
levels = {r["value"]: int(r["n"]) for r in self._rows(
|
|
f"SELECT roles.seniority AS value, COUNT(*) AS n {by_level} "
|
|
"GROUP BY roles.seniority", level_params)}
|
|
pays = {r["value"]: int(r["n"]) for r in self._rows(
|
|
f"SELECT {_PAY_STATE} AS value, COUNT(*) AS n {by_pay} GROUP BY value",
|
|
pay_params)}
|
|
return {"seniority": [(v, levels.get(v, 0)) for v in (*LEVELS, "")],
|
|
"pay": [(v, pays.get(v, 0)) for v in ("normalised", "stated", "")]}
|
|
|
|
def _roles_from(self, *, company: str | None, place: str | None, open_only: bool,
|
|
q: str | None, function: str | None, remote: str | None,
|
|
days: int | None, hide_muted: bool, include_read: bool,
|
|
tags: Sequence[str], tag_mode: str, agent: int | None,
|
|
by_score: bool, gated: bool, judged: str | None,
|
|
pay: str | None = None, company_keys: Sequence[str] = (),
|
|
places: Sequence[str] = (), seniority: Sequence[str] = (),
|
|
seniority_unknown: bool = True,
|
|
pay_unknown: bool = False,
|
|
closing: bool = False) -> tuple[str, list[Any]]:
|
|
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
|
its parameters in order. The agent's votes are always joined as `v`
|
|
(see `_WITH_VOTES`) and its judgements as `j` (`_WITH_JUDGEMENTS`);
|
|
its scores as `s` only when ordering by them. A copy of another role
|
|
(`rebuild_dupes`) is never listed: its kept role stands for it and
|
|
counts its copies. `closing` keeps the roles whose deadline is
|
|
today or within `CLOSING_DAYS` after it."""
|
|
where: list[str] = ["roles.dup_of IS NULL"]
|
|
params: list[Any] = []
|
|
if company:
|
|
where.append("company LIKE ? ESCAPE '\\'")
|
|
params.append(_like(company))
|
|
if place:
|
|
where.append("place LIKE ? ESCAPE '\\'")
|
|
params.append(_like(place))
|
|
if function:
|
|
where.append("function = ?")
|
|
params.append(function)
|
|
if remote:
|
|
where.append("remote = ?")
|
|
params.append(remote)
|
|
if days is not None:
|
|
where.append("posted_at >= ?")
|
|
params.append(_iso(self.clock() - timedelta(days=days)))
|
|
for word in (q or "").split():
|
|
where.append("(title LIKE ? ESCAPE '\\' OR company LIKE ? ESCAPE '\\')")
|
|
params += [_like(word)] * 2
|
|
if judged:
|
|
if judged not in JUDGED:
|
|
raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}")
|
|
where.append(JUDGED[judged])
|
|
if pay:
|
|
if pay not in PAY:
|
|
raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}")
|
|
where.append(f"({PAY[pay]} OR {_NO_PAY})" if pay_unknown else PAY[pay])
|
|
if company_keys:
|
|
where.append(f"company_key IN ({', '.join('?' for _ in company_keys)})")
|
|
params += list(company_keys)
|
|
if seniority:
|
|
if unknown := sorted(set(seniority) - set(LEVELS)):
|
|
raise ValueError(f"seniority is {', '.join(LEVELS)}, not {unknown[0]!r}")
|
|
levels = f"roles.seniority IN ({', '.join('?' for _ in seniority)})"
|
|
where.append(f"({levels} OR roles.seniority = '')" if seniority_unknown else levels)
|
|
params += list(seniority)
|
|
if open_only:
|
|
where.append("closed_at IS NULL")
|
|
if hide_muted and agent is not None:
|
|
where.append("company_key NOT IN (SELECT company_key FROM agent_companies "
|
|
"WHERE agent_id = ? AND affinity = -1)")
|
|
params.append(agent)
|
|
if not include_read:
|
|
where.append("roles.read_at IS NULL")
|
|
if closing:
|
|
today = self.clock().date()
|
|
where.append("roles.deadline BETWEEN ? AND ?")
|
|
params += [today.isoformat(), (today + timedelta(days=CLOSING_DAYS)).isoformat()]
|
|
if tags:
|
|
marks = ", ".join("?" for _ in tags)
|
|
if tag_mode == "all":
|
|
where.append(
|
|
f"(SELECT COUNT(DISTINCT rt.tag) FROM role_tags rt "
|
|
f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks})) = ?")
|
|
params += [*tags, len(tags)]
|
|
elif tag_mode == "any":
|
|
where.append(
|
|
f"EXISTS (SELECT 1 FROM role_tags rt "
|
|
f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks}))")
|
|
params += list(tags)
|
|
else:
|
|
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
|
|
if places:
|
|
_known_places(places)
|
|
clause, values = _places_sql(places, self.gazetteer(), scope=self._scope(agent))
|
|
where.append(clause)
|
|
params += values
|
|
sql = f"FROM roles {_WITH_VOTES} {_WITH_JUDGEMENTS}"
|
|
joins: list[Any] = [agent, agent]
|
|
if agent is not None and by_score:
|
|
sql += " JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?"
|
|
joins.append(agent)
|
|
if not gated:
|
|
where.append("s.gated_reason = ''")
|
|
if where:
|
|
sql += " WHERE " + " AND ".join(where)
|
|
return sql, [*joins, *params]
|
|
|
|
def _scope(self, agent_id: int | None) -> list[Gate]:
|
|
"""The agent's `require` gates on country: what `remote:any` keeps a
|
|
remote role within. None without an agent."""
|
|
found = self.agent(agent_id) if agent_id is not None else None
|
|
return [g for g in found.gates if g.field == "country" and g.mode == "require"] \
|
|
if found is not None else []
|
|
|
|
def tag_counts(self, agent_id: int | None = None,
|
|
show_muted: bool = False) -> list[tuple[str, int, int]]:
|
|
"""`(tag, open_count, unread_count)` over open roles, most-used tag
|
|
first and then alphabetically. What `/api/filters` serves; the
|
|
companies `agent_id` hid are left out unless `show_muted`, the same
|
|
default as `roles`, and with no agent nothing is hidden."""
|
|
where = "r.closed_at IS NULL AND r.dup_of IS NULL"
|
|
params: list[Any] = []
|
|
if not show_muted and agent_id is not None:
|
|
where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
|
|
"WHERE agent_id = ? AND affinity = -1)")
|
|
params.append(agent_id)
|
|
rows = self._rows(
|
|
f"""SELECT rt.tag AS tag, COUNT(*) AS n, SUM(r.read_at IS NULL) AS unread
|
|
FROM role_tags rt JOIN roles r ON r.uid = rt.uid
|
|
WHERE {where}
|
|
GROUP BY rt.tag
|
|
ORDER BY n DESC, rt.tag ASC""", params)
|
|
return [(str(r["tag"]), int(r["n"]), int(r["unread"])) for r in rows]
|
|
|
|
def mark_closed(self, source: str, seen_ids: Sequence[str],
|
|
subscription_id: int) -> int:
|
|
"""Close every open role of one board that a complete fetch of it did not
|
|
list. Returns how many closed. Only call it after a fetch that finished:
|
|
a failed one lists nothing, and would close the whole board."""
|
|
return self._write(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
|
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
|
|
|
|
def close_unseen(self, source: str, before: str) -> int:
|
|
"""Close every open role of `source` last seen before `before`. For a
|
|
source whose fetch is never a complete list, a search, where a role
|
|
dropping out of one run's results is not the role closing."""
|
|
return self._write(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND closed_at IS NULL AND last_seen < ?""",
|
|
(self.now(), source, before))
|
|
|
|
def close_subscription(self, subscription_id: int) -> int:
|
|
"""Close every open role of one subscription, for a board that is no
|
|
longer fetched: nothing would ever close them otherwise."""
|
|
return self._write(
|
|
"UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL",
|
|
(self.now(), subscription_id))
|
|
|
|
def backfill_from_place(self) -> int:
|
|
"""Fill `country` and `remote` from `place` on every role that has
|
|
none, the way `upsert_role` does for a new one, so a word added to the
|
|
lookups reaches the roles already stored. Returns how many columns it
|
|
filled."""
|
|
rows = self._rows("SELECT uid, place FROM roles "
|
|
"WHERE place != '' AND (country = '' OR remote = '')")
|
|
countries = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))]
|
|
remotes = [(mode, r["uid"]) for r in rows if (mode := remote_of(r["place"]))]
|
|
with self.transaction() as conn:
|
|
conn.executemany("UPDATE roles SET country = ? WHERE uid = ? AND country = ''",
|
|
countries)
|
|
conn.executemany("UPDATE roles SET remote = ? WHERE uid = ? AND remote = ''",
|
|
remotes)
|
|
return len(countries) + len(remotes)
|
|
|
|
# ----------------------------------------------------------------- prep
|
|
|
|
def prep_rows(self, scope: Literal["due", "open", "all"] = "due",
|
|
uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
|
|
"""What `app.prep` maps a role from, and what it holds now: the roles
|
|
due (`prep_due` above 0), every open role, or every role; of `uids`
|
|
only, when given. `remote_area` is the role's `role_remote` as a JSON
|
|
list, in order."""
|
|
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
|
|
params: list[Any] = []
|
|
if uids is not None:
|
|
where += " AND uid IN (SELECT value FROM json_each(?))"
|
|
params.append(json.dumps(list(uids)))
|
|
return self._rows(
|
|
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
|
seniority_host, employment_host, remote_host, country_host,
|
|
seniority_model, seniority_stated, seniority_title, seniority,
|
|
employment, remote, country, remote_model, remote_where_model,
|
|
deadline_host, deadline_model, deadline,
|
|
json_extract(raw, '$.places') AS raw_places,
|
|
(SELECT json_group_array(place_id) FROM (
|
|
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
|
ORDER BY rr.rowid)) AS remote_area
|
|
FROM roles WHERE {where}""", params)
|
|
|
|
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
|
|
"""Resolve these roles' own places against the gazetteer, writing the
|
|
rows that resolve differently; `uid -> place ids` for each that has
|
|
places (`gazetteer.relink_uids`)."""
|
|
with self.transaction() as conn:
|
|
return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids))
|
|
|
|
def company_folds(self) -> dict[str, list[str]]:
|
|
"""`fold -> company keys` of `company_aliases`, sorted: what
|
|
`resolve_company` asks one name at a time, for prep's whole batch."""
|
|
out: dict[str, list[str]] = {}
|
|
for r in self._rows("SELECT fold, company_key FROM company_aliases "
|
|
"ORDER BY fold, company_key"):
|
|
out.setdefault(str(r["fold"]), []).append(str(r["company_key"]))
|
|
return out
|
|
|
|
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
|
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
|
`prep_due` it read), `company_key`, `seniority_stated`,
|
|
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
|
|
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
|
the title's. `remote` is written as prep composed it (the model's,
|
|
else the host's word, else the place line) and `remote_area`, place
|
|
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
|
column as it is, since `extract` fills an empty one. `prep_due` goes
|
|
back to 0 only when no pull changed the role since it was read.
|
|
Returns the rows written."""
|
|
with self.transaction() as conn:
|
|
written = conn.executemany(
|
|
"""UPDATE roles SET company_key = :company_key,
|
|
seniority_stated = :seniority_stated,
|
|
seniority_title = :seniority_title,
|
|
seniority = COALESCE(NULLIF(seniority_model, ''),
|
|
NULLIF(:seniority_stated, ''), :seniority_title),
|
|
employment = :employment,
|
|
remote = :remote,
|
|
country = COALESCE(NULLIF(:country, ''), country),
|
|
deadline = :deadline,
|
|
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
|
WHERE uid = :uid""", mapped).rowcount
|
|
conn.executemany("DELETE FROM role_remote WHERE uid = ?",
|
|
[(m["uid"],) for m in mapped])
|
|
conn.executemany("INSERT OR IGNORE INTO role_remote (uid, place_id) VALUES (?, ?)",
|
|
[(m["uid"], p) for m in mapped for p in m.get("remote_area", ())])
|
|
return written
|
|
|
|
def host_words(self) -> list[tuple[str, str, str, int]]:
|
|
"""`(field, host, word, open roles)` for every host word an open role
|
|
carries, the word lowercased: what prep checks against its tables."""
|
|
found: list[tuple[str, str, str, int]] = []
|
|
for field in ("seniority", "employment", "remote", "country"):
|
|
found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows(
|
|
f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles
|
|
WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")]
|
|
return found
|
|
|
|
def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None:
|
|
"""Make `unmapped` these `(field, host, word, roles)`, keeping the
|
|
`first_seen` of a word already listed."""
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute(
|
|
"SELECT field, host, word, first_seen FROM unmapped")}
|
|
conn.execute("DELETE FROM unmapped")
|
|
conn.executemany(
|
|
"INSERT INTO unmapped (field, host, word, roles, first_seen) "
|
|
"VALUES (?, ?, ?, ?, ?)",
|
|
[(*r, seen.get(r[:3], stamp)) for r in rows])
|
|
|
|
def unmapped(self) -> list[dict[str, Any]]:
|
|
"""The host words no table maps, by field and host, most roles first."""
|
|
return [dict(r) for r in self._rows(
|
|
"SELECT field, host, word, roles, first_seen FROM unmapped "
|
|
"ORDER BY field, host, roles DESC, word")]
|
|
|
|
# -------------------------------------------------------------- extract
|
|
|
|
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
|
"""The `WHERE` fragment for a role among the best ungated scores of
|
|
an enabled agent, as deep and as low as that agent's own `limits`
|
|
say for `stage`: what the two model stages, extract and summarise,
|
|
are bounded to. The rank is numbered per agent and the agent's
|
|
limits ride out of the window subquery beside it, so each row is
|
|
held to the limits of the agent that ranked it; a key the JSON lacks
|
|
is the `Limits` default. The role is `r`. A company an agent hid
|
|
needs no clause: `score` writes no rows for it under that agent. An
|
|
agent without a prompt with text counts for nothing here: its scores
|
|
have no fit, so they say how fresh a role is, not whether it is
|
|
wanted."""
|
|
defaults = Limits()
|
|
return (f"""r.uid IN (SELECT uid FROM (
|
|
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
|
|
PARTITION BY s.agent_id
|
|
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
|
FROM scores s
|
|
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
|
AND EXISTS (SELECT 1 FROM agent_prompts p
|
|
WHERE p.agent_id = a.id
|
|
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
|
|
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
|
AND top.dup_of IS NULL
|
|
WHERE s.gated_reason = '')
|
|
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
|
|
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
|
|
[getattr(defaults, f"{stage}_top"), getattr(defaults, f"{stage}_min")])
|
|
|
|
def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
|
"""The `WHERE` for a role `extract` may read: open, never answered,
|
|
not given up. Given `uids`, those of them and nothing else, since a
|
|
batch asks again when it runs and must finish what it was dealt;
|
|
otherwise only the roles in `_top_scope`."""
|
|
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
|
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
|
|
.replace("dup_of", "r.dup_of")
|
|
.replace("seniority_model", "r.seniority_model")
|
|
.replace("remote_where_model", "r.remote_where_model")
|
|
.replace("languages_required", "r.languages_required")]
|
|
if uids is not None:
|
|
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
|
return " AND ".join(where), [json.dumps(list(uids))]
|
|
top, params = self._top_scope("extract")
|
|
return " AND ".join([*where, top]), params
|
|
|
|
def extract_pending(self) -> int:
|
|
where, params = self._extract_scope(None)
|
|
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
|
|
return int(row["n"]) if row else 0
|
|
|
|
def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
|
"""Roles `extract` has yet to read: the best of every enabled agent's
|
|
scores, as deep and as low as that agent's own `limits` say, so the
|
|
backlog follows the rankings rather than the boards. Ordered by the
|
|
best rank any agent gives the role (each agent's first, then each
|
|
agent's second ...), then by its score, so every agent's top comes
|
|
before anyone's tail whatever limits cut the batch. Given `uids`,
|
|
only those of them still waiting, whatever their rank; a role no agent
|
|
ranks comes last."""
|
|
where, params = self._extract_scope(uids)
|
|
return [Role.from_row(r) for r in self._rows(
|
|
"SELECT r.* FROM roles r LEFT JOIN ("
|
|
" SELECT uid, MIN(n) AS best_rank, MAX(score) AS best_score FROM ("
|
|
" SELECT s.uid, s.score, ROW_NUMBER() OVER ("
|
|
" PARTITION BY s.agent_id ORDER BY s.score DESC, s.uid) AS n"
|
|
" FROM scores s"
|
|
" JOIN agents a ON a.id = s.agent_id AND a.enabled = 1"
|
|
" WHERE s.gated_reason = '') GROUP BY uid) k ON k.uid = r.uid "
|
|
f"WHERE {where} "
|
|
"ORDER BY k.best_rank IS NULL, k.best_rank, k.best_score DESC, "
|
|
"r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
|
|
|
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
|
|
"""Write what `extract` read for one role and stamp `extracted_at`.
|
|
|
|
Only into empty columns, and only a non-empty value: a board's own
|
|
field is never replaced and nothing is ever blanked. `place` brings
|
|
`place_key` and `place_fold` with it. Returns how many columns were filled, or None
|
|
for an unknown uid. Values are trusted to be in vocabulary; the
|
|
job's `apply` is where a reply is checked.
|
|
|
|
`function` is the one exception to "only into empty columns": it is
|
|
the `ats` classifier's keyword guess, not a board field, so a
|
|
non-empty answer overwrites it outright rather than only filling an
|
|
empty column — see `upsert_role`'s docstring for the other half of
|
|
the rule that then keeps a later pull from putting the guess back.
|
|
`seniority` is the other: a non-empty answer is `seniority_model`,
|
|
which comes before the stated and the title's level, so it becomes
|
|
`seniority` too. `remote` and `remote_where` are written outright as
|
|
`remote_model` and `remote_where_model` (a JSON list, `[]` for none),
|
|
even when empty, since an answer is what the model read this time;
|
|
`remote` and the area are composed from them by prep, so the role is
|
|
marked due (`prep.run_roles`, which `extract` calls).
|
|
`languages_required` (a JSON list, `[]` for none) and the deadline,
|
|
as `deadline_model` (`YYYY-MM-DD` or ''), are written outright too;
|
|
prep composes `deadline` from the host's and this one."""
|
|
with self.transaction() as conn:
|
|
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
sets: dict[str, Any] = {}
|
|
for name in _EXTRACT_TEXT:
|
|
value = str(fields.get(name) or "")
|
|
if value and not row[name]:
|
|
sets[name] = value
|
|
function = str(fields.get("function") or "")
|
|
if function:
|
|
sets["function"] = function
|
|
level = str(fields.get("seniority") or "")
|
|
if level:
|
|
sets["seniority_model"] = sets["seniority"] = level
|
|
for name in _RAW_INT:
|
|
number = _int_or_none(fields.get(name))
|
|
if number is not None and row[name] is None:
|
|
sets[name] = number
|
|
languages = [str(code) for code in fields.get("languages") or []]
|
|
if languages and json.loads(row["languages"] or "[]") == []:
|
|
sets["languages"] = json.dumps(languages)
|
|
remote = str(fields.get("remote") or "")
|
|
where = [str(name) for name in fields.get("remote_where") or []]
|
|
named = list(fields.get("languages_required") or [])
|
|
deadline = str(fields.get("deadline") or "")
|
|
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where) \
|
|
+ bool(named) + bool(deadline)
|
|
sets["remote_model"] = remote
|
|
sets["remote_where_model"] = json.dumps(where)
|
|
sets["languages_required"] = json.dumps(named)
|
|
sets["deadline_model"] = deadline
|
|
if "place" in sets:
|
|
sets["place_key"] = place_key(sets["place"])
|
|
sets["place_fold"] = places.fold(sets["place"])
|
|
gazetteer.write(conn, self._gazetteer(conn), uid,
|
|
gazetteer.texts({"place": sets["place"]}))
|
|
sets["extracted_at"] = self.now()
|
|
conn.execute(
|
|
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)}, "
|
|
"prep_due = prep_due + 1 WHERE uid=?", (*sets.values(), uid))
|
|
return filled
|
|
|
|
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
|
|
"""One reply that left these roles unanswered. Counted only against
|
|
those still waiting, so a role the reply did answer is not charged.
|
|
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
|
|
for `ModelTask.missed` and not kept: there is no Given up list yet."""
|
|
if not uids:
|
|
return 0
|
|
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
|
|
wanted = json.dumps(list(uids))
|
|
with self.transaction() as conn:
|
|
crossed = conn.execute(
|
|
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
|
|
(wanted, GIVE_UP_AFTER)).fetchone()[0]
|
|
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
|
|
f"WHERE {where}", (wanted,))
|
|
return int(crossed)
|
|
|
|
def seniority_agreement(self) -> list[dict[str, Any]]:
|
|
"""How far each host's stated level matches the model's, over the
|
|
open roles that have both: `host`, `compared`, `agreed` and `rate`
|
|
(agreed / compared), the least trusted host first. `stated_only`
|
|
counts the roles that state a level and have no model value yet, so
|
|
a small `compared` can be told from a host that states nothing."""
|
|
rows = self._rows(
|
|
f"SELECT {_HOST} AS host, "
|
|
"SUM(seniority_stated <> '' AND seniority_model <> '') AS compared, "
|
|
"SUM(seniority_stated <> '' AND seniority_stated = seniority_model) AS agreed, "
|
|
"SUM(seniority_stated <> '' AND seniority_model = '') AS stated_only "
|
|
"FROM roles WHERE closed_at IS NULL AND dup_of IS NULL "
|
|
"GROUP BY host HAVING compared > 0 OR stated_only > 0")
|
|
found = [{"host": r["host"], "compared": int(r["compared"]),
|
|
"agreed": int(r["agreed"]), "stated_only": int(r["stated_only"]),
|
|
"rate": round(r["agreed"] / r["compared"], 3) if r["compared"] else None}
|
|
for r in rows]
|
|
return sorted(found, key=lambda h: (h["rate"] is None, h["rate"] or 0, h["host"]))
|
|
|
|
# ------------------------------------------------------------ summarise
|
|
|
|
#: The `state` key counting replies that left a role without a summary.
|
|
#: `roles` has no column for it, and the count only matters for the few
|
|
#: roles in some agent's top list.
|
|
_SUMMARY_MISSED = "summarise.missed:"
|
|
|
|
def _summary_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
|
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
|
|
some text to summarise. Given `uids`, those of them and nothing else,
|
|
open or not and given up or not, because somebody asked for them.
|
|
Otherwise open, not given up, and in `_top_scope` or one the owner
|
|
cares about whatever its rank: saved (it has an application) or voted
|
|
up by an enabled agent."""
|
|
where = ["r.summary = ''",
|
|
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
|
if uids is not None:
|
|
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
|
return " AND ".join(where), [json.dumps(list(uids))]
|
|
top, params = self._top_scope("summarise")
|
|
where += [
|
|
"r.closed_at IS NULL",
|
|
"r.dup_of IS NULL",
|
|
f"""NOT EXISTS (SELECT 1 FROM state
|
|
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
|
|
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
|
|
f"""({top}
|
|
OR r.uid IN (SELECT role_uid FROM applications
|
|
WHERE role_uid IS NOT NULL)
|
|
OR r.uid IN (SELECT v.uid FROM role_votes v
|
|
JOIN agents a ON a.id = v.agent_id AND a.enabled = 1
|
|
WHERE v.vote = 1))""",
|
|
]
|
|
return " AND ".join(where), params
|
|
|
|
def summarise_pending(self) -> int:
|
|
where, params = self._summary_scope(None)
|
|
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
|
|
return int(row["n"]) if row else 0
|
|
|
|
def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
|
"""Roles `summarise` has yet to write, newest first: the top of every
|
|
enabled agent's scores, as deep and as low as that agent's own
|
|
`limits` say, so the backlog is bounded by the agents rather than by
|
|
the number of open roles, and every open role that is saved or voted
|
|
up by an enabled agent, whatever its rank; given `uids`, those still without a
|
|
summary, whatever their rank. A batch asks again when it runs."""
|
|
where, params = self._summary_scope(uids)
|
|
return [Role.from_row(r) for r in self._rows(
|
|
f"SELECT r.* FROM roles r WHERE {where} "
|
|
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
|
|
|
def set_summary(self, uid: str, summary: str, tags: Sequence[str] = ()) -> bool:
|
|
"""Write a role's summary and stamp `summarised_at`. Only over an empty
|
|
one, so a second batch that carried the same role does not replace the
|
|
first answer. Given `tags`, they replace the role's tags in the same
|
|
transaction — see `_replace_tags` — but only when the summary was
|
|
actually written, so a rejected late answer cannot plant tags on a
|
|
role it did not get to summarise. False for an unknown uid or one
|
|
already summarised."""
|
|
with self.transaction() as conn:
|
|
written = conn.execute(
|
|
"UPDATE roles SET summary = ?, summarised_at = ? "
|
|
"WHERE uid = ? AND summary = ''", (summary, self.now(), uid)).rowcount
|
|
conn.execute("DELETE FROM state WHERE key = ?", (self._SUMMARY_MISSED + uid,))
|
|
if written and tags:
|
|
self._replace_tags(conn, uid, tags)
|
|
return bool(written)
|
|
|
|
def summarise_missed(self, uids: Sequence[str], why: str = "") -> int:
|
|
"""One reply that left these roles without a summary. Counted only
|
|
against those still without one. Returns how many this pushed to
|
|
`GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept."""
|
|
if not uids:
|
|
return 0
|
|
rows = self._rows("SELECT uid FROM roles WHERE summary = '' "
|
|
"AND uid IN (SELECT value FROM json_each(?))",
|
|
(json.dumps(list(uids)),))
|
|
keys = [self._SUMMARY_MISSED + r["uid"] for r in rows]
|
|
with self.transaction() as conn:
|
|
conn.executemany(
|
|
"""INSERT INTO state (key, value) VALUES (?, '1')
|
|
ON CONFLICT(key) DO UPDATE
|
|
SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)""",
|
|
[(key,) for key in keys])
|
|
crossed = conn.execute(
|
|
"SELECT COUNT(*) FROM state WHERE key IN (SELECT value FROM json_each(?)) "
|
|
"AND CAST(value AS INTEGER) = ?", (json.dumps(keys), GIVE_UP_AFTER)).fetchone()[0]
|
|
return int(crossed)
|
|
|
|
# ---------------------------------------------------------------- judge
|
|
|
|
def judge_rows(self, agent: Agent, uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
|
|
"""What `judge` may read for one agent: its best ungated scores of
|
|
open roles that are no copy, as deep and as low as its `limits` say
|
|
(`judge_top`, `judge_min`), best first, then the `judge_cv_top`
|
|
ungated ones closest to its CV (`scores.cv_fit`) that the first set
|
|
leaves out, closest first, each with some text (`article_text` or
|
|
`description`). Given `uids`, those of them with
|
|
text, whatever their rank or state. Each row carries `uid`,
|
|
`title`, `company`, `place`, `article_text`, `description`, and the
|
|
stored judgement's `input_key`, `judged_at` and `tries` (NULL, NULL
|
|
and 0 without one), and `given_up` once `tries` reached
|
|
`GIVE_UP_AFTER`."""
|
|
text = "(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"
|
|
select = ("SELECT r.uid, r.title, r.company, r.place, r.article_text, r.description, "
|
|
"j.input_key, j.judged_at, COALESCE(j.tries, 0) AS tries, "
|
|
f"COALESCE(j.tries, 0) >= {GIVE_UP_AFTER} AS given_up FROM ")
|
|
joined = "LEFT JOIN role_judgements j ON j.uid = r.uid AND j.agent_id = ?"
|
|
if uids is not None:
|
|
return self._rows(
|
|
f"{select} roles r {joined} WHERE r.uid IN (SELECT value FROM json_each(?)) "
|
|
f"AND {text} ORDER BY r.uid", (agent.id, json.dumps(list(uids))))
|
|
ranked = ("FROM scores s JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL "
|
|
"AND top.dup_of IS NULL WHERE s.agent_id = ? AND s.gated_reason = ''")
|
|
return self._rows(
|
|
f"""WITH best AS (SELECT s.uid, s.score {ranked}
|
|
ORDER BY s.score DESC, top.posted_at DESC, s.uid LIMIT ?),
|
|
by_score AS (SELECT uid, 0 AS part, score AS rank FROM best
|
|
WHERE score >= ?),
|
|
by_cv AS (SELECT s.uid, 1 AS part, s.cv_fit AS rank {ranked}
|
|
AND s.cv_fit IS NOT NULL
|
|
AND s.uid NOT IN (SELECT uid FROM by_score)
|
|
ORDER BY s.cv_fit DESC, top.posted_at DESC, s.uid LIMIT ?)
|
|
{select} (SELECT * FROM by_score UNION ALL SELECT * FROM by_cv) w
|
|
JOIN roles r ON r.uid = w.uid {joined}
|
|
WHERE {text}
|
|
ORDER BY w.part, w.rank DESC, r.posted_at DESC, r.uid""",
|
|
(agent.id, agent.limits.judge_top, agent.limits.judge_min, agent.id,
|
|
agent.limits.judge_cv_top, agent.id))
|
|
|
|
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
|
|
experience: int, career: int, strengths: Sequence[str],
|
|
gaps: Sequence[str], musts: Sequence[Mapping[str, str]] = ()) -> bool:
|
|
"""Store `judge`'s reading of one role for one agent, replacing the
|
|
one before: the three numbers as the model gave them, the verdict on
|
|
each of the agent's must-haves (`musts`, `{id, text, verdict}`) and how
|
|
many are broken, `cv_score` (`app.models.cv_score`, at the default
|
|
weights and with the cap: the table's column is not read, see
|
|
`judgement`) and its tries back at 0. False for an unknown role or
|
|
agent."""
|
|
broken = sum(1 for m in musts if m.get("verdict") == "broken")
|
|
try:
|
|
with self.transaction() as conn:
|
|
conn.execute(
|
|
"""INSERT INTO role_judgements (uid, agent_id, skills, experience, career,
|
|
cv_score, strengths, gaps, musts, broken,
|
|
input_key, tries, judged_at)
|
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,0,?)
|
|
ON CONFLICT(uid, agent_id) DO UPDATE SET
|
|
skills=excluded.skills, experience=excluded.experience,
|
|
career=excluded.career, cv_score=excluded.cv_score,
|
|
strengths=excluded.strengths, gaps=excluded.gaps,
|
|
musts=excluded.musts, broken=excluded.broken,
|
|
input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""",
|
|
(uid, agent_id, skills, experience, career,
|
|
cv_score(skills, experience, career, broken=broken),
|
|
json.dumps(list(strengths), ensure_ascii=False),
|
|
json.dumps(list(gaps), ensure_ascii=False),
|
|
json.dumps([dict(m) for m in musts], ensure_ascii=False), broken,
|
|
input_key, self.now()))
|
|
except sqlite3.IntegrityError:
|
|
return False
|
|
return True
|
|
|
|
def judge_missed(self, pairs: Sequence[tuple[int, str]]) -> int:
|
|
"""One reply that left these `(agent_id, uid)` without a judgement:
|
|
each is charged a try, a row of tries alone made for one never
|
|
judged. Returns how many this pushed to `GIVE_UP_AFTER`. The caller
|
|
passes only the pairs still waiting."""
|
|
if not pairs:
|
|
return 0
|
|
with self.transaction() as conn:
|
|
conn.executemany(
|
|
"""INSERT INTO role_judgements (uid, agent_id, tries)
|
|
SELECT ?, ?, 1 WHERE EXISTS (SELECT 1 FROM roles WHERE uid = ?)
|
|
ON CONFLICT(uid, agent_id) DO UPDATE SET tries = tries + 1""",
|
|
[(uid, agent_id, uid) for agent_id, uid in pairs])
|
|
crossed = sum(conn.execute(
|
|
"SELECT COUNT(*) FROM role_judgements WHERE uid = ? AND agent_id = ? "
|
|
"AND tries = ?", (uid, agent_id, GIVE_UP_AFTER)).fetchone()[0]
|
|
for agent_id, uid in pairs)
|
|
return int(crossed)
|
|
|
|
def judgement(self, agent_id: int | None, uid: str) -> Judgement | None:
|
|
"""The agent's judgement of one role, None when it has none, its
|
|
`cv_score` the three numbers under the agent's weights as they are
|
|
now, not as stored, and `career_capped` the model's career as the
|
|
broken must-haves leave it (`BROKEN_CAP`)."""
|
|
if agent_id is None:
|
|
return None
|
|
row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? "
|
|
"AND judged_at IS NOT NULL", (agent_id, uid))
|
|
if row is None:
|
|
return None
|
|
found = Judgement.from_row(row)
|
|
agent = self.agent(agent_id)
|
|
found.career_capped = capped_career(found.career, found.broken)
|
|
found.cv_score = cv_score(found.skills, found.experience, found.career,
|
|
agent.weights if agent else None, found.broken)
|
|
return found
|
|
|
|
def set_article_text(self, uid: str, text: str) -> Role | None:
|
|
"""Store the posting page as plain text and stamp `article_at`. The
|
|
role as it now stands, or None for an unknown uid."""
|
|
if not self._write("UPDATE roles SET article_text = ?, article_at = ? WHERE uid = ?",
|
|
(text, self.now(), uid)):
|
|
return None
|
|
return self.role(uid)
|
|
|
|
def set_posting_url(self, uid: str, url: str) -> bool:
|
|
"""Keep where a search host's own page leads. False for an unknown
|
|
uid. Only this writes it: `upsert_role` leaves it alone, so a pull
|
|
keeps it."""
|
|
return bool(self._write("UPDATE roles SET posting_url = ? WHERE uid = ?", (url, uid)))
|
|
|
|
# --------------------------------------------------------------- enrich
|
|
|
|
def enrich_waiting(self, top: int, floor: float,
|
|
uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
|
|
"""Open board and search roles without text that an enabled agent with a prompt
|
|
ranks ungated among its best `top` and at `floor` or above, best
|
|
first by the highest such score, as `{uid, source_id,
|
|
subscription_id, title, url, published_at, raw, score}`. The ranking
|
|
is `_top_scope`'s, with the bar given rather than the agent's own.
|
|
Given `uids`, those of them still without text, whatever their rank,
|
|
in no order (`score` is None), since a batch finishes what it was
|
|
dealt."""
|
|
fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, "
|
|
"r.published_at, r.raw")
|
|
waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source IN ('ats', 'search') "
|
|
"AND r.subscription_id IS NOT NULL AND r.description = '' "
|
|
"AND r.article_text = ''")
|
|
if uids is not None:
|
|
rows = self._rows(
|
|
f"SELECT {fields}, NULL AS score FROM roles r WHERE {waiting} "
|
|
"AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),))
|
|
else:
|
|
rows = self._rows(
|
|
f"""SELECT {fields}, MAX(ranked.score) AS score FROM (
|
|
SELECT s.uid, s.score, ROW_NUMBER() OVER (
|
|
PARTITION BY s.agent_id
|
|
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
|
FROM scores s
|
|
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
|
AND EXISTS (SELECT 1 FROM agent_prompts p
|
|
WHERE p.agent_id = a.id
|
|
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
|
|
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
|
AND top.dup_of IS NULL
|
|
WHERE s.gated_reason = '') ranked
|
|
JOIN roles r ON r.uid = ranked.uid
|
|
WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting}
|
|
GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor))
|
|
return [dict(row) for row in rows]
|
|
|
|
# -------------------------------------------------------------- vectors
|
|
|
|
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
|
"""Open roles with no vector from `model`, newest first, so a backlog
|
|
embedded from the top improves what is on screen first. A copy is
|
|
not embedded: it is never scored."""
|
|
return [Role.from_row(r) for r in self._rows(
|
|
"""SELECT r.* FROM roles r
|
|
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
|
|
WHERE v.uid IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL
|
|
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
|
|
|
|
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
return conn.executemany(
|
|
"""INSERT INTO vectors (uid, model, dim, vec, embedded_at)
|
|
VALUES (?,?,?,?,?)
|
|
ON CONFLICT(uid) DO UPDATE SET model=excluded.model,
|
|
dim=excluded.dim, vec=excluded.vec,
|
|
embedded_at=excluded.embedded_at""",
|
|
[(uid, model, int(vec.shape[-1]), pack(vec), stamp)
|
|
for uid, vec in pairs]).rowcount
|
|
|
|
def vectors(self, model: str) -> dict[str, np.ndarray]:
|
|
"""Every vector from `model`, by uid. The whole table: at ~30k roles of
|
|
384 floats that is ~45 MB, and one read beats one per agent."""
|
|
return {r["uid"]: unpack(r["vec"]) for r in self._rows(
|
|
"SELECT uid, vec FROM vectors WHERE model = ?", (model,))}
|
|
|
|
def vector_counts(self) -> dict[str, int]:
|
|
return {r["model"]: int(r["n"]) for r in self._rows(
|
|
"SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")}
|
|
|
|
def drop_vectors(self, keep: str) -> int:
|
|
"""Delete vectors from every model but `keep`: another model's are a
|
|
different geometry, never compared, and only make counts lie."""
|
|
return self._write("DELETE FROM vectors WHERE model <> ?", (keep,))
|
|
|
|
# --------------------------------------------------------------- agents
|
|
|
|
def agents(self, enabled_only: bool = False) -> list[Agent]:
|
|
sql = "SELECT * FROM agents"
|
|
if enabled_only:
|
|
sql += " WHERE enabled = 1"
|
|
return self._agents(self._rows(sql + " ORDER BY name COLLATE NOCASE"))
|
|
|
|
def agent(self, agent_id: int) -> Agent | None:
|
|
found = self._agents(self._rows("SELECT * FROM agents WHERE id=?", (agent_id,)))
|
|
return found[0] if found else None
|
|
|
|
def default_agent(self) -> Agent | None:
|
|
"""The agent a vote or an affinity goes to when none is named: the
|
|
enabled one with the lowest id, else the lowest id, else None."""
|
|
found = self._agents(self._rows(
|
|
"SELECT * FROM agents ORDER BY enabled DESC, id LIMIT 1"))
|
|
return found[0] if found else None
|
|
|
|
def _agents(self, rows: Sequence[sqlite3.Row]) -> list[Agent]:
|
|
"""The rows as agents, each with its prompts in order, read with one
|
|
query for them all."""
|
|
prompts: dict[int, list[dict[str, str]]] = {}
|
|
for p in self._rows(
|
|
"SELECT agent_id, name, text FROM agent_prompts WHERE agent_id IN "
|
|
"(SELECT value FROM json_each(?)) ORDER BY agent_id, position",
|
|
(json.dumps([r["id"] for r in rows]),)):
|
|
prompts.setdefault(p["agent_id"], []).append({"name": p["name"], "text": p["text"]})
|
|
return [Agent.from_row({**dict(r), "prompts": prompts.get(r["id"], [])})
|
|
for r in rows]
|
|
|
|
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
|
|
weights: Weights | None = None, enabled: bool = True,
|
|
agent_id: int | None = None, limits: Limits | None = None,
|
|
prompts: Sequence[Prompt] | None = None,
|
|
fetch_terms: Sequence[str] | None = None,
|
|
cv_text: str | None = None,
|
|
cv_languages: Sequence[CvLanguage] | None = None,
|
|
avoid_text: str | None = None,
|
|
must_text: str | None = None,
|
|
like_text: str | None = None) -> Agent:
|
|
"""Create an agent, or with `agent_id` replace that one's fields and
|
|
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
|
|
named after the agent. Without `fetch_terms` an update keeps the
|
|
stored ones and a new agent has none, so a caller that does not know
|
|
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
|
|
`like_text`, `avoid_text` and `must_text` are kept the same way. A prompt whose text is
|
|
unchanged keeps its cached vector, whatever its name or place, so an
|
|
edit re-embeds only the prompts it changed. A save gives every role
|
|
`judge` gave up on for the agent its tries back, since what it is
|
|
judged against may have changed. Raises ValueError on an empty or
|
|
taken name, or two prompts of one name, KeyError on an unknown id."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("an agent needs a name")
|
|
if prompts is None:
|
|
prompts = [Prompt(name=name, text=intent)] if intent.strip() else []
|
|
prompts = check_prompts(list(prompts))
|
|
terms = None if fetch_terms is None else json.dumps(
|
|
normal_terms(list(fetch_terms)), ensure_ascii=False)
|
|
spoken = None if cv_languages is None else json.dumps(
|
|
[lang.model_dump() for lang in normal_languages(list(cv_languages))])
|
|
fields = (name, json.dumps([g.model_dump() for g in gates]),
|
|
(weights or Weights()).model_dump_json(),
|
|
(limits or Limits()).model_dump_json(), int(enabled), terms,
|
|
None if cv_text is None else cv_text.strip(), spoken,
|
|
None if avoid_text is None else avoid_text.strip(),
|
|
None if must_text is None else must_text.strip(),
|
|
None if like_text is None else like_text.strip())
|
|
try:
|
|
with self.transaction() as conn:
|
|
if agent_id is None:
|
|
agent_id = conn.execute(
|
|
"""INSERT INTO agents (name, gates, weights, limits, enabled,
|
|
fetch_terms, cv_text, cv_languages, avoid_text,
|
|
must_text, like_text, created_at)
|
|
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
|
|
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),
|
|
COALESCE(?, ''),?)""",
|
|
(*fields, self.now())).lastrowid
|
|
elif not conn.execute(
|
|
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
|
|
fetch_terms=COALESCE(?, fetch_terms),
|
|
cv_text=COALESCE(?, cv_text),
|
|
cv_languages=COALESCE(?, cv_languages),
|
|
avoid_text=COALESCE(?, avoid_text),
|
|
must_text=COALESCE(?, must_text),
|
|
like_text=COALESCE(?, like_text)
|
|
WHERE id=?""",
|
|
(*fields, agent_id)).rowcount:
|
|
raise KeyError(agent_id)
|
|
conn.execute("UPDATE role_judgements SET tries = 0 "
|
|
"WHERE agent_id = ? AND tries > 0", (agent_id,))
|
|
cached = {r["text"]: (r["vec_key"], r["vec"]) for r in conn.execute(
|
|
"SELECT text, vec_key, vec FROM agent_prompts WHERE agent_id=?",
|
|
(agent_id,))}
|
|
conn.execute("DELETE FROM agent_prompts WHERE agent_id=?", (agent_id,))
|
|
conn.executemany(
|
|
"""INSERT INTO agent_prompts (agent_id, position, name, text, vec_key, vec)
|
|
VALUES (?,?,?,?,?,?)""",
|
|
[(agent_id, n, p.name, p.text, *cached.get(p.text, ("", None)))
|
|
for n, p in enumerate(prompts)])
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"an agent named {name!r} already exists") from exc
|
|
assert agent_id is not None
|
|
found = self.agent(agent_id)
|
|
assert found is not None
|
|
return found
|
|
|
|
def delete_agent(self, agent_id: int) -> bool:
|
|
"""Its prompts, scores, votes and affinities go with it, by the
|
|
foreign keys."""
|
|
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
|
|
|
|
def prompt_vector(self, agent_id: int, name: str, key: str) -> np.ndarray | None:
|
|
"""The cached vector of the agent's prompt `name`, if it was made
|
|
under `key`."""
|
|
row = self._row("SELECT vec_key, vec FROM agent_prompts WHERE agent_id=? AND name=?",
|
|
(agent_id, name))
|
|
if row is None or row["vec_key"] != key or row["vec"] is None:
|
|
return None
|
|
return unpack(row["vec"])
|
|
|
|
def set_prompt_vector(self, agent_id: int, name: str, key: str, vec: np.ndarray) -> None:
|
|
self._write("UPDATE agent_prompts SET vec_key=?, vec=? WHERE agent_id=? AND name=?",
|
|
(key, pack(vec), agent_id, name))
|
|
|
|
def cv_vectors(self, agent_id: int, key: str) -> list[np.ndarray] | None:
|
|
"""The cached vectors of the agent's CV, one per chunk in order, if
|
|
they were made under `key`; None otherwise."""
|
|
rows = self._rows("SELECT vec_key, vec FROM agent_cv_vectors WHERE agent_id=? "
|
|
"ORDER BY position", (agent_id,))
|
|
if not rows or any(r["vec_key"] != key for r in rows):
|
|
return None
|
|
return [unpack(r["vec"]) for r in rows]
|
|
|
|
def set_cv_vectors(self, agent_id: int, key: str, vectors: Sequence[np.ndarray]) -> None:
|
|
"""Replace the agent's CV vectors with these, made under `key`. An
|
|
agent deleted meanwhile gets nothing written, as in `replace_scores`."""
|
|
with self.transaction() as conn:
|
|
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
|
|
return
|
|
conn.execute("DELETE FROM agent_cv_vectors WHERE agent_id=?", (agent_id,))
|
|
conn.executemany(
|
|
"INSERT INTO agent_cv_vectors (agent_id, position, vec_key, vec) "
|
|
"VALUES (?,?,?,?)",
|
|
[(agent_id, n, key, pack(vec)) for n, vec in enumerate(vectors)])
|
|
|
|
# --------------------------------------------------------------- scores
|
|
|
|
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
|
|
"""Every open role of a company this agent has not hidden, copies
|
|
(`rebuild_dupes`) aside, with the agent's affinity for the company
|
|
and how the role meets each gate: 1 match, 0 no match, -1 the column
|
|
is empty. The gates are evaluated here, in SQL, so a `require` gate is
|
|
the `WHERE` it reads as, written
|
|
as a column so a gated role keeps its row and reason."""
|
|
columns: list[str] = []
|
|
params: list[Any] = []
|
|
gaz = self.gazetteer()
|
|
for n, gate in enumerate(gates):
|
|
# An exclude gate says which of its values matched: one WHEN each.
|
|
parts = ([gate.model_copy(update={"values": [v]}) for v in gate.values]
|
|
if gate.mode == "exclude" else [gate])
|
|
whens = []
|
|
for k, part in enumerate(parts, 1):
|
|
empty, match, values = _gate_sql(part, gaz)
|
|
whens.append(f"WHEN {match} THEN {k}")
|
|
params += values
|
|
columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}")
|
|
select = ", ".join(["r.uid", "r.posted_at",
|
|
"COALESCE(ac.affinity, 0) AS affinity", *columns])
|
|
rows = self._rows(
|
|
f"""SELECT {select} FROM roles r
|
|
LEFT JOIN agent_companies ac
|
|
ON ac.company_key = r.company_key AND ac.agent_id = ?
|
|
WHERE r.closed_at IS NULL AND r.dup_of IS NULL
|
|
AND COALESCE(ac.affinity, 0) <> -1""",
|
|
[*params, agent_id])
|
|
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
|
|
affinity=int(r["affinity"]),
|
|
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
|
for r in rows]
|
|
|
|
def replace_scores(self, agent_id: int,
|
|
rows: Sequence[tuple[str, float, str] | tuple[str, float, str, str]
|
|
| tuple[str, float, str, str, float | None]]
|
|
) -> int:
|
|
"""`(uid, score, gated_reason)` for every role this agent scored,
|
|
with the name of the prompt that gave the fit as a fourth item ('' when
|
|
left out) and the role's closeness to the agent's CV as a fifth (None
|
|
when left out), replacing whatever it had: a role closed or muted since the last run
|
|
drops out rather than keeping a stale score. One transaction, so a
|
|
reader sees the old list or the new one. An agent deleted while its
|
|
run was scoring gets nothing written and 0 back: its rows are gone
|
|
with it, and an insert would only fail the foreign key."""
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
|
|
return 0
|
|
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
|
|
conn.executemany(
|
|
"""INSERT INTO scores (uid, agent_id, score, gated_reason, prompt, cv_fit,
|
|
scored_at)
|
|
VALUES (?,?,?,?,?,?,?)""",
|
|
[(row[0], agent_id, row[1], row[2], row[3] if len(row) > 3 else "",
|
|
row[4] if len(row) > 4 else None, stamp)
|
|
for row in rows])
|
|
return len(rows)
|
|
|
|
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
|
|
"""`uid -> (score, gated_reason)` for one agent."""
|
|
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
|
|
for r in self._rows(
|
|
"SELECT uid, score, gated_reason FROM scores WHERE agent_id=?",
|
|
(agent_id,))}
|
|
|
|
# ------------------------------------------------------------ companies
|
|
|
|
def add_company(self, name: str, *, key: str = "", domain: str = "",
|
|
status: str = "researching", careers_url: str = "",
|
|
notes: str = "", ats: str = "", country: str = "",
|
|
city: str = "", ats_slug: str = "",
|
|
hand_reason: str = "") -> Company:
|
|
"""Raises ValueError on a key already taken or a `hand_reason` outside
|
|
`HAND_REASONS`."""
|
|
name = name.strip()
|
|
key = key or company_key(name)
|
|
if not key:
|
|
raise ValueError("a company needs a name")
|
|
_check_hand_reason(hand_reason)
|
|
try:
|
|
with self.transaction() as conn:
|
|
self._insert_company(conn, key, name, {
|
|
"domain": domain, "status": status, "careers_url": careers_url,
|
|
"notes": notes, "ats": ats, "country": country, "city": city,
|
|
"ats_slug": ats_slug, "hand_reason": hand_reason})
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"company {key!r} already exists") from exc
|
|
return self._company(key)
|
|
|
|
def _insert_company(self, conn: sqlite3.Connection, key: str, name: str,
|
|
fields: dict[str, Any]) -> None:
|
|
stamp = self.now()
|
|
columns = ["key", "name", *fields, "created_at", "updated_at"]
|
|
conn.execute(
|
|
f"INSERT INTO companies ({', '.join(columns)}) "
|
|
f"VALUES ({', '.join('?' * len(columns))})",
|
|
(key, name, *fields.values(), stamp, stamp))
|
|
self._alias(conn, key, name)
|
|
|
|
def company(self, key: str, agent_id: int | None = None) -> Company | None:
|
|
"""One company, with `affinity` as `agent_id` set it; 0 without an agent."""
|
|
row = self._row(f"{_COMPANY_SELECT} WHERE c.key=?", (agent_id, key))
|
|
return Company.from_row(row) if row else None
|
|
|
|
def companies(self, agent_id: int | None = None) -> list[Company]:
|
|
"""Every company by name, with `affinity` as `agent_id` set it."""
|
|
return [Company.from_row(r) for r in self._rows(
|
|
f"{_COMPANY_SELECT} ORDER BY c.name COLLATE NOCASE", (agent_id,))]
|
|
|
|
def set_affinity(self, agent_id: int, key: str, affinity: int) -> Company:
|
|
"""One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
|
|
favourite. 0 deletes the row, since no row means neutral. Returns the
|
|
company with this agent's affinity. Raises ValueError for another
|
|
value and KeyError for an unknown company or agent.
|
|
|
|
A key with roles and no company row, an employer known only from a
|
|
search, gets a row here holding only its key and the name its roles
|
|
give it, the most frequent one: nothing else can be judged without
|
|
one. It has no board, `ats`, `careers_url` or `domain`, which is how
|
|
the Companies list tells it from a followed company."""
|
|
_check_affinity(affinity)
|
|
with self.transaction() as conn:
|
|
self._known(conn, "agents", "id", agent_id)
|
|
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
|
|
named = conn.execute(
|
|
"""SELECT company FROM roles WHERE company_key=? AND company<>''
|
|
GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC
|
|
LIMIT 1""", (key,)).fetchone()
|
|
if named is None:
|
|
raise KeyError(key)
|
|
self._insert_company(conn, key, named[0], {})
|
|
if affinity == 0:
|
|
conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?",
|
|
(agent_id, key))
|
|
else:
|
|
conn.execute(
|
|
"""INSERT INTO agent_companies (agent_id, company_key, affinity, updated_at)
|
|
VALUES (?,?,?,?)
|
|
ON CONFLICT(agent_id, company_key) DO UPDATE
|
|
SET affinity=excluded.affinity, updated_at=excluded.updated_at""",
|
|
(agent_id, key, affinity, self.now()))
|
|
return self._company(key, agent_id)
|
|
|
|
def update_company(self, key: str, **fields: Any) -> Company:
|
|
"""Change any of `_COMPANY_FIELDS`. Affinity is per agent:
|
|
`set_affinity`."""
|
|
unknown = set(fields) - set(_COMPANY_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
|
|
if "hand_reason" in fields:
|
|
_check_hand_reason(fields["hand_reason"])
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
with self.transaction() as conn:
|
|
changed = conn.execute(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
|
(*fields.values(), self.now(), key)).rowcount
|
|
if not changed:
|
|
raise KeyError(key)
|
|
if "name" in fields:
|
|
self._alias(conn, key, fields["name"])
|
|
return self._company(key)
|
|
|
|
def set_all_countries(self, key: str, on: bool) -> Company:
|
|
"""Switch a company's All countries override: its big boards are read
|
|
with the agents' wide reach, Europe today, rather than their places,
|
|
from the next pull (`app/jobs/reach.py`). Global, not per agent.
|
|
Raises KeyError for an unknown company."""
|
|
return self.update_company(key, all_countries=int(bool(on)))
|
|
|
|
def set_all_roles(self, key: str, on: bool) -> Company:
|
|
"""Switch a company's All roles override: its boards are read with no
|
|
role-type narrowing, no fetch terms today, from the next pull
|
|
(`app/jobs/reach.py`). Global, not per agent. Raises KeyError for an
|
|
unknown company."""
|
|
return self.update_company(key, all_roles=int(bool(on)))
|
|
|
|
def remove_company(self, key: str) -> bool:
|
|
"""Delete the row and every agent's stance on it, in one transaction:
|
|
`agent_companies` holds the key with no foreign key to hold it back, so
|
|
a row left behind would give a company added again under the same name
|
|
an affinity it never got. Roles and subscriptions are not touched."""
|
|
with self.transaction() as conn:
|
|
conn.execute("DELETE FROM agent_companies WHERE company_key=?", (key,))
|
|
conn.execute("DELETE FROM company_aliases WHERE company_key=?", (key,))
|
|
return conn.execute("DELETE FROM companies WHERE key=?", (key,)).rowcount > 0
|
|
|
|
# ---------------------------------------------------- company aliases
|
|
|
|
@staticmethod
|
|
def _alias(conn: sqlite3.Connection, key: str, name: str) -> None:
|
|
if fold := company_fold(name):
|
|
conn.execute("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)",
|
|
(fold, key))
|
|
|
|
def resolve_company(self, name: str) -> str:
|
|
"""The key a role from `name` belongs to: the company an alias names,
|
|
the one with the name's own key first when several share the fold,
|
|
else `company_key(name)`. `Orsted` lands on the `ørsted` company."""
|
|
own = company_key(name)
|
|
row = self._row("SELECT company_key FROM company_aliases WHERE fold=? "
|
|
"ORDER BY company_key=? DESC, company_key LIMIT 1",
|
|
(company_fold(name), own))
|
|
return str(row["company_key"]) if row else own
|
|
|
|
def add_alias(self, key: str, name: str) -> None:
|
|
"""Another name for company `key`: a search and a role from `name` find
|
|
it. Raises KeyError for an unknown company, ValueError for a name that
|
|
folds to nothing."""
|
|
if not company_fold(name):
|
|
raise ValueError("an alias needs a name")
|
|
with self.transaction() as conn:
|
|
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
|
|
raise KeyError(key)
|
|
self._alias(conn, key, name)
|
|
|
|
def company_aliases(self) -> dict[str, list[str]]:
|
|
"""`company_key -> folds`, for the company search."""
|
|
out: dict[str, list[str]] = {}
|
|
for r in self._rows("SELECT company_key, fold FROM company_aliases ORDER BY fold"):
|
|
out.setdefault(str(r["company_key"]), []).append(str(r["fold"]))
|
|
return out
|
|
|
|
def duplicate_companies(self) -> list[tuple[str, list[str]]]:
|
|
"""`(fold, keys)` for each fold more than one company has, likely the
|
|
same company under two spellings. Nothing is merged."""
|
|
return [(str(r["fold"]), str(r["keys"]).split("\t")) for r in self._rows(
|
|
"SELECT fold, group_concat(company_key, char(9)) AS keys FROM company_aliases "
|
|
"GROUP BY fold HAVING COUNT(*) > 1 ORDER BY fold")]
|
|
|
|
def merge_companies(self, keep: str, other: str) -> Company:
|
|
"""Fold company `other` into `keep`, in one transaction: its roles and
|
|
subscriptions move to `keep`, each agent's stance moves unless the
|
|
agent has one on `keep` already, the All countries and All roles
|
|
overrides carry over, its aliases and its name become aliases of
|
|
`keep`, and its row goes. Copies are not recomputed (`rebuild_dupes`).
|
|
Raises KeyError for an unknown company and ValueError for merging one
|
|
into itself."""
|
|
if keep == other:
|
|
raise ValueError("a company cannot be merged into itself")
|
|
with self.transaction() as conn:
|
|
rows = {r["key"]: r["name"] for r in conn.execute(
|
|
"SELECT key, name FROM companies WHERE key IN (?, ?)", (keep, other))}
|
|
for key in (keep, other):
|
|
if key not in rows:
|
|
raise KeyError(key)
|
|
conn.execute("UPDATE roles SET company_key=? WHERE company_key=?", (keep, other))
|
|
for sub in conn.execute("SELECT id, company FROM subscriptions").fetchall():
|
|
if company_key(sub["company"]) == other:
|
|
conn.execute("UPDATE subscriptions SET company=? WHERE id=?",
|
|
(rows[keep], sub["id"]))
|
|
conn.execute("UPDATE OR IGNORE agent_companies SET company_key=? WHERE company_key=?",
|
|
(keep, other))
|
|
conn.execute("DELETE FROM agent_companies WHERE company_key=?", (other,))
|
|
conn.execute("UPDATE OR IGNORE company_aliases SET company_key=? WHERE company_key=?",
|
|
(keep, other))
|
|
conn.execute("DELETE FROM company_aliases WHERE company_key=?", (other,))
|
|
self._alias(conn, keep, rows[other])
|
|
conn.execute("UPDATE companies SET "
|
|
"all_roles = MAX(all_roles, (SELECT all_roles FROM companies WHERE key=?)), "
|
|
"all_countries = MAX(all_countries, "
|
|
"(SELECT all_countries FROM companies WHERE key=?)) WHERE key=?",
|
|
(other, other, keep))
|
|
conn.execute("DELETE FROM companies WHERE key=?", (other,))
|
|
return self._company(keep)
|
|
|
|
def _company(self, key: str, agent_id: int | None = None) -> Company:
|
|
found = self.company(key, agent_id)
|
|
if found is None:
|
|
raise KeyError(key)
|
|
return found
|
|
|
|
def open_role_counts(self) -> dict[str, int]:
|
|
"""`company_key -> open role count`, for the company list's row
|
|
count. A key with no open roles is simply absent. Copies are not
|
|
counted, so the number is the rows the feed lists for the company."""
|
|
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
|
|
"SELECT company_key, COUNT(*) AS n FROM roles "
|
|
"WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")}
|
|
|
|
def open_companies(self, agent_id: int | None = None
|
|
) -> list[tuple[str, list[str], int, int]]:
|
|
"""`(company_key, names, open roles, affinity)` for every company with
|
|
an open role, what the Company field suggests from. `names` are the
|
|
company row's name and then every spelling its roles carry, commonest
|
|
first, so a search finds a company by any of them. A company
|
|
`agent_id` hid is left out; `affinity` is that agent's, 0 without
|
|
one. Copies are not counted, as in the feed."""
|
|
rows = self._rows(
|
|
"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n,
|
|
c.name AS own, COALESCE(ac.affinity, 0) AS affinity
|
|
FROM roles r
|
|
LEFT JOIN companies c ON c.key = r.company_key
|
|
LEFT JOIN agent_companies ac ON ac.company_key = r.company_key
|
|
AND ac.agent_id = ?
|
|
WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != ''
|
|
AND COALESCE(ac.affinity, 0) != -1
|
|
GROUP BY r.company_key, r.company
|
|
ORDER BY r.company_key, n DESC, r.company""", (agent_id,))
|
|
found: dict[str, tuple[list[str], list[int], int]] = {}
|
|
for r in rows:
|
|
names, counts, _ = found.setdefault(
|
|
r["key"], ([r["own"]] if r["own"] else [], [], int(r["affinity"])))
|
|
if r["name"] and r["name"] not in names:
|
|
names.append(r["name"])
|
|
counts.append(int(r["n"]))
|
|
return [(key, names, sum(counts), affinity)
|
|
for key, (names, counts, affinity) in found.items()]
|
|
|
|
def open_places(self) -> list[tuple[str, str, int]]:
|
|
"""`(place, country, open roles)` for every place line an open role
|
|
carries, what `places.suggest` reads. Copies are not counted, as in
|
|
the feed."""
|
|
return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows(
|
|
"SELECT place, country, COUNT(*) AS n FROM roles "
|
|
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
|
|
"GROUP BY place, country")]
|
|
|
|
def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]:
|
|
"""`(area, open roles)` for every remote area an open remote role
|
|
holds, the area as place ids in order: what `places.suggest` counts
|
|
the remote tokens from. Copies are not counted, as in the feed."""
|
|
found: dict[str, list[int]] = {}
|
|
for r in self._rows(
|
|
"SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid "
|
|
"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' "
|
|
"ORDER BY rr.uid, rr.rowid"):
|
|
found.setdefault(str(r[0]), []).append(int(r[1]))
|
|
return list(Counter(tuple(ids) for ids in found.values()).items())
|
|
|
|
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
|
"""Per subscription id, its open roles counted by `country`, copies
|
|
included: what a board or a search holds, not what the feed lists."""
|
|
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
|
|
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
|
GROUP BY subscription_id, country""")
|
|
out: dict[int, dict[str, int]] = {}
|
|
for r in rows:
|
|
out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"]
|
|
return out
|
|
|
|
def company_boards(self) -> dict[str, str]:
|
|
"""`company_key -> ats host`, read off each company's own
|
|
subscription. `Subscription.company` is the display name it and the
|
|
company row were both added with, so `company_key` of one finds the
|
|
other; a company with no subscription, or a detached one (`show` 0,
|
|
see `set_company_board`), is simply absent."""
|
|
boards: dict[str, str] = {}
|
|
for sub in self.subscriptions(source="ats"):
|
|
if sub.show:
|
|
boards.setdefault(company_key(sub.company), sub.board)
|
|
return boards
|
|
|
|
def company_subscription(self, key: str) -> Subscription | None:
|
|
"""Company `key`'s own `ats` subscription, detached or not; None when
|
|
it never had one. A company has at most one: `set_company_board`
|
|
rewrites it in place rather than adding a second."""
|
|
with closing(self._connect()) as conn:
|
|
return self._company_subscription(conn, key)
|
|
|
|
@staticmethod
|
|
def _company_subscription(conn: sqlite3.Connection, key: str) -> Subscription | None:
|
|
rows = conn.execute("SELECT * FROM subscriptions WHERE source = 'ats' "
|
|
"ORDER BY show DESC, id").fetchall()
|
|
return next((Subscription.from_row(r) for r in rows
|
|
if company_key(r["company"]) == key), None)
|
|
|
|
def set_company_board(self, key: str, target: str, *, host: str = "", name: str = "",
|
|
fields: dict[str, Any] | None = None,
|
|
) -> tuple[Company, Subscription | None, int]:
|
|
"""Point company `key` at the `ats` board `target`, or at none when
|
|
`target` is "", in one transaction, so a clash on either table leaves
|
|
nothing half written.
|
|
|
|
A missing company is created from `name` and `fields`; an existing one
|
|
has `fields` written over it. Its subscription is rewritten in place:
|
|
a new target closes the old target's open roles, as switching a board
|
|
off does, and "" detaches it (off, hidden, its roles closed) while the
|
|
row stays for the next board. Returns the company, its subscription
|
|
(None when there is none) and how many roles closed. Raises KeyError
|
|
for a missing company with no board, ValueError on a UNIQUE clash."""
|
|
fields = dict(fields or {})
|
|
unknown = set(fields) - set(_COMPANY_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
|
|
stamp = self.now()
|
|
closed = 0
|
|
try:
|
|
with self.transaction() as conn:
|
|
row = conn.execute("SELECT name FROM companies WHERE key=?", (key,)).fetchone()
|
|
if row is None:
|
|
name = fields.pop("name", "") or name.strip()
|
|
if not target:
|
|
raise KeyError(key)
|
|
if not name:
|
|
raise ValueError("a company needs a name")
|
|
self._insert_company(conn, key, name, fields)
|
|
else:
|
|
name = fields.get("name") or row["name"]
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
conn.execute(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
|
(*fields.values(), stamp, key))
|
|
sub = self._company_subscription(conn, key)
|
|
sub_id = sub.id if sub else None
|
|
if sub is not None and sub.target != target:
|
|
closed = conn.execute(
|
|
"UPDATE roles SET closed_at = ? "
|
|
"WHERE subscription_id = ? AND closed_at IS NULL",
|
|
(stamp, sub.id)).rowcount
|
|
if not target:
|
|
if sub is not None:
|
|
conn.execute("UPDATE subscriptions SET subscribed=0, fetch=0, show=0 "
|
|
"WHERE id=?", (sub.id,))
|
|
elif sub is not None:
|
|
conn.execute(
|
|
"""UPDATE subscriptions
|
|
SET target=?, board=?, label=?, company=?, subscribed=1,
|
|
fetch=1, show=1,
|
|
target_id=CASE WHEN target=? THEN target_id ELSE '' END
|
|
WHERE id=?""",
|
|
(target, host, name, name, target, sub.id))
|
|
else:
|
|
sub_id = conn.execute(
|
|
"""INSERT INTO subscriptions (source, target, label, company, board,
|
|
params, added_at)
|
|
VALUES ('ats',?,?,?,?,'{}',?)""",
|
|
(target, name, name, host, stamp)).lastrowid
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"another board already has {target!r} or the label "
|
|
f"{name!r}") from exc
|
|
return (self._company(key), self.subscription(sub_id) if sub_id else None, closed)
|
|
|
|
# -------------------------------------------------------- subscriptions
|
|
|
|
def subscriptions(self, source: str | None = None,
|
|
fetchable: bool = False) -> list[Subscription]:
|
|
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
|
|
where, params = [], []
|
|
if source:
|
|
where.append("source = ?")
|
|
params.append(source)
|
|
if fetchable:
|
|
where.append("fetch = 1 AND subscribed = 1")
|
|
sql = "SELECT * FROM subscriptions"
|
|
if where:
|
|
sql += " WHERE " + " AND ".join(where)
|
|
sql += " ORDER BY company, board, label"
|
|
return [Subscription.from_row(r) for r in self._rows(sql, params)]
|
|
|
|
def subscription(self, sub_id: int) -> Subscription | None:
|
|
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
|
|
return Subscription.from_row(row) if row else None
|
|
|
|
def subscription_for(self, source: str, label: str) -> Subscription | None:
|
|
"""The row an item names: `Item.subscription` is the label."""
|
|
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
|
|
(source, label))
|
|
return Subscription.from_row(row) if row else None
|
|
|
|
def add_subscription(self, source: str, target: str, label: str = "",
|
|
company: str = "", board: str = "",
|
|
params: dict[str, Any] | None = None,
|
|
target_id: str = "") -> Subscription:
|
|
"""Raises ValueError on a duplicate target or label within one plugin."""
|
|
target = target.strip()
|
|
if not target:
|
|
raise ValueError("a subscription needs a target")
|
|
label = label.strip() or target
|
|
try:
|
|
with self.transaction() as conn:
|
|
new_id = conn.execute(
|
|
"""INSERT INTO subscriptions (source, target, label, company, board,
|
|
params, added_at, target_id)
|
|
VALUES (?,?,?,?,?,?,?,?)""",
|
|
(source, target, label, company, board, json.dumps(params or {}),
|
|
self.now(), target_id)).lastrowid
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
|
|
assert new_id is not None
|
|
return self._subscription(new_id)
|
|
|
|
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
|
|
"""Change labels, params, target_id or the three switches."""
|
|
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
|
|
if "params" in fields:
|
|
fields["params"] = json.dumps(fields["params"] or {})
|
|
for switch in ("subscribed", "fetch", "show"):
|
|
if switch in fields:
|
|
fields[switch] = int(bool(fields[switch]))
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
try:
|
|
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
|
|
(*fields.values(), sub_id))
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
|
|
if not changed:
|
|
raise KeyError(sub_id)
|
|
return self._subscription(sub_id)
|
|
|
|
def unsubscribe(self, sub_id: int) -> Subscription:
|
|
"""Stop following it and hide it. The row stays, and so do its roles."""
|
|
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
|
|
|
|
def _subscription(self, sub_id: int) -> Subscription:
|
|
found = self.subscription(sub_id)
|
|
if found is None:
|
|
raise KeyError(sub_id)
|
|
return found
|
|
|
|
# --------------------------------------------------------- applications
|
|
|
|
def save_application(self, application: Application) -> Application:
|
|
"""Insert or update one application. An empty `id` is minted. Entering a
|
|
status, on insert or by change, appends an `application_events` row in
|
|
the same transaction."""
|
|
_check_status(application.status)
|
|
app_id = application.id or uuid4().hex
|
|
stamp = self.now()
|
|
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
|
|
with self.transaction() as conn:
|
|
before = conn.execute("SELECT status FROM applications WHERE id=?",
|
|
(app_id,)).fetchone()
|
|
if before is None:
|
|
columns = ["id", *fields, "created_at", "updated_at"]
|
|
conn.execute(
|
|
f"INSERT INTO applications ({', '.join(columns)}) "
|
|
f"VALUES ({', '.join('?' for _ in columns)})",
|
|
(app_id, *fields.values(), stamp, stamp))
|
|
else:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
|
|
(*fields.values(), stamp, app_id))
|
|
if before is None or before["status"] != application.status:
|
|
conn.execute("INSERT INTO application_events (application_id, status, at) "
|
|
"VALUES (?,?,?)", (app_id, application.status, stamp))
|
|
return self._application(app_id)
|
|
|
|
def set_application_status(self, app_id: str, status: str) -> Application:
|
|
"""Move one application to `status`. Raises ValueError for a status
|
|
outside `STATUSES` and KeyError for an unknown id."""
|
|
_check_status(status)
|
|
current = self._application(app_id)
|
|
return self.save_application(current.model_copy(update={"status": status}))
|
|
|
|
def application(self, app_id: str) -> Application | None:
|
|
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
|
|
return Application.from_row(row) if row else None
|
|
|
|
def delete_application(self, app_id: str) -> bool:
|
|
"""Drop one application and its events. Whether that is allowed (only
|
|
while it is still `saved`) is the caller's rule, not the store's."""
|
|
return self._write("DELETE FROM applications WHERE id=?", (app_id,)) > 0
|
|
|
|
def applications(self, status: str | None = None) -> list[Application]:
|
|
sql, params = "SELECT * FROM applications", []
|
|
if status:
|
|
sql += " WHERE status = ?"
|
|
params.append(status)
|
|
sql += " ORDER BY updated_at DESC, id"
|
|
return [Application.from_row(r) for r in self._rows(sql, params)]
|
|
|
|
def application_events(self, app_id: str) -> list[ApplicationEvent]:
|
|
return [ApplicationEvent.from_row(r) for r in self._rows(
|
|
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
|
|
(app_id,))]
|
|
|
|
def _application(self, app_id: str) -> Application:
|
|
found = self.application(app_id)
|
|
if found is None:
|
|
raise KeyError(app_id)
|
|
return found
|
|
|
|
# ---------------------------------------------------------------- state
|
|
|
|
@property
|
|
def state(self) -> MutableMapping[str, str]:
|
|
"""The `state` table as a mapping. What the runner hands a plugin as
|
|
`self.state`: it can keep a counter without being able to read roles."""
|
|
return _StateMap(self)
|
|
|
|
def state_items(self, prefix: str) -> dict[str, str]:
|
|
"""Every `state` key that starts with `prefix`, with its value, in one
|
|
query, for a reader that wants one kind of key across every target."""
|
|
return {str(r["key"]): str(r["value"]) for r in self._rows(
|
|
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
|
|
(len(prefix), prefix))}
|
|
|
|
def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool:
|
|
"""Set `key` to `value` only while it is free: unset, equal to the
|
|
value under `released` (the claim last let go of), or no later than
|
|
`stale`. Returns whether it was set. One statement, so of two
|
|
processes claiming at once exactly one wins."""
|
|
return self._write(
|
|
"""INSERT INTO state (key, value) VALUES (?, ?)
|
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
WHERE state.value <= ?
|
|
OR state.value = COALESCE(
|
|
(SELECT r.value FROM state AS r WHERE r.key = ?), '')""",
|
|
(key, value, stale, released)) > 0
|
|
|
|
|
|
def _check_status(status: str) -> None:
|
|
if status not in STATUSES:
|
|
raise ValueError(f"{status!r} is not an application status "
|
|
f"({', '.join(STATUSES)})")
|
|
|
|
|
|
def _check_hand_reason(reason: Any) -> None:
|
|
if reason not in HAND_REASONS:
|
|
raise ValueError(f"hand_reason is one of {', '.join(repr(r) for r in HAND_REASONS)}, "
|
|
f"not {reason!r}")
|
|
|
|
|
|
def _check_affinity(affinity: Any) -> None:
|
|
if isinstance(affinity, bool) or not isinstance(affinity, int) or affinity not in AFFINITY:
|
|
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")
|