diff --git a/docs/decisions.md b/docs/decisions.md new file mode 100644 index 0000000..3c2b9e4 --- /dev/null +++ b/docs/decisions.md @@ -0,0 +1,45 @@ +# Key decisions + +What was decided for roledesk and the one reason for each. The longer reasoning is in `../news/docs/JobBoardPlan/roles-build-plan.md` and `jobhunt-hq-assessment.md`. + +## Shape + +**Own repo, depending on news for `desk` and `pipeline`.** `pyproject.toml` takes `newsdesk` as an editable path dependency on `../news`. `desk` imports no app and news's architecture test enforces it, so it works as a library. Copying it would fork the plugin contract; `pipeline` alone may be copied later if depending on a repo named news becomes the problem. + +**`desk` and `pipeline` are never edited here.** A change roles needs lands in news, and roledesk picks it up through the editable install. + +**The domain noun is `role`.** `jobs/` keeps meaning the work the app queues, so "job" never has two senses in one sentence. + +**Own database, queue and port.** `roledesk.db`, `roles.yaml`, port 8790. A shared file with newsdesk would buy "articles about this company" by cosine, which is worth less than two apps that can migrate independently now that `Jobs` takes a path. + +**Storage is written, not imported.** `desk` has no storage package on purpose, so ~120 lines of connection plumbing are copied from newsdesk. If the same edit lands in both a second time, a shared `Connection` moves into `desk` below `pipeline`. + +**Layers are `models, config, storage, jobs, web, cli`.** `tests/test_architecture.py` holds the direction, the doors, and that a plugin imports `desk` only, since a plugin is app-agnostic. + +## Data + +**Clustering key is `(company_key, title_key, place_key)`.** newsdesk's `location_key OR title_key` collapses two outlets running one story, which is right for wire copy and wrong for ads, where the same title at two companies is two jobs. + +**`place`, `country` and `remote` are separate columns.** `location` already means where an `Item`'s original lives. `remote` alone over-matches, because boards write "UK | Remote", so a gate on remote also reads `country`. + +**`raw` is transport, columns are storage.** No `Item.facets`. A plugin puts structured fields in `raw` under the column's name and `upsert_role` unpacks them into indexed columns, which is where gates query them. An empty value never overwrites a filled column, so a terse board does not erase what `extract` found. + +**Postings are persisted.** Storing every posting is what gives `first_seen`, which is the only honest posted date since Greenhouse resets `updated_at` on every edit, and `closed_at`, set by `mark_closed` when a complete fetch of a board no longer lists a role. A role listed again is open again. + +**The tracker schema is taken from jobhunt-hq (MIT).** `applications` with status in `saved, applied, screen, technical, onsite, offer, rejected, withdrawn`, `next_action`, `next_date`, and an append-only `application_events` written by the store on every status entered. The events table is what "days to first screen" and conversion by source are computed from. + +**`subscriptions` names its labels `company` and `board`.** newsdesk's `outlet` and `section` are news words; roledesk has no rows to migrate, so it uses its own names from the first migration. + +## Ranking + +**Scoring is per search agent.** A role is scored as `(uid, agent) -> score`, because a role is only good for a given search. Each gate has a mode: `require` becomes a SQL `WHERE`, `prefer` becomes a weighted term in the score. Gated roles keep their row with a reason. `companies.affinity = 2` skips the gates and floors the score, `-1` mutes. + +**`fit` is cosine between the role and the agent's intent text.** The intent is a CV or a paragraph per agent. This is why the embedding backend is effectively required: TF-IDF cannot relate "Kubernetes" to "container orchestration". + +**Summary and tagging are off; `extract` replaces them.** Every profile declares `SummaryPolicy(enabled=False)` and `TaggingPolicy(enabled=False)`. `extract` is a roledesk `ModelTask` on Haiku that turns title and description into the gate columns, and skips rows a plugin already filled from structured fields. + +## Outside the app + +**Unposted roles and the network stay outside.** Their state is inside: an application row with no `role_uid`, a `contact`, `next_action` and `next_date`, and a company `status` such as researching or contacted. + +**LinkedIn enters only as alert mail over IMAP.** Scraping it is banned by contract and enforced. The alert mails also carry personalised recommendations no public API exposes. diff --git a/roledesk/models.py b/roledesk/models.py index 4db7813..950a738 100644 --- a/roledesk/models.py +++ b/roledesk/models.py @@ -1,3 +1,165 @@ -"""The shapes roledesk hands round. Filled in with the storage layer.""" +"""The shapes roledesk hands round, and the keys that decide two rows are one role. + +One set of models, as in newsdesk: the row the table returns and the object +the API serves are the same declaration. `Model.from_row` ignores columns a +model does not declare, so a new column is never a 500 somewhere else. + +The three keys cluster roles on `(company_key, title_key, place_key)`. The same +title at two companies is two jobs, and the same ad on two boards is one. +""" from __future__ import annotations + +import re +import unicodedata +from typing import Literal, get_args + +from desk.models import JsonDict, Model + +# ------------------------------------------------------------------- keys + +#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x), +#: (w/m/div), (all genders), (gn). Matched with or without the parentheses. +_GENDER = re.compile( + r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)" + r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)" + r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b", + re.IGNORECASE, +) + +#: Legal forms cut from the end of a company name, so "Celonis SE" on one +#: board and "celonis" on another share a key. +_LEGAL = re.compile( + r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|" + r"co|bv|b\.v|sa|sas|srl|oy|ab|as|aps))+\.?$", + re.IGNORECASE, +) + + +def _fold(text: str) -> str: + """Casefolded, accents stripped, punctuation to spaces, whitespace collapsed.""" + text = unicodedata.normalize("NFKD", text.casefold()) + text = "".join(c for c in text if not unicodedata.combining(c)) + text = re.sub(r"[\W_]+", " ", text) + return " ".join(text.split()) + + +def title_key(title: str) -> str: + """`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`.""" + return _fold(_GENDER.sub(" ", title or "")) + + +def place_key(place: str) -> str: + """`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`.""" + return _fold(place or "") + + +def company_key(company: str) -> str: + """A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the + primary key of `companies`.""" + name = _LEGAL.sub("", (company or "").strip()) + return _fold(name).replace(" ", "-") + + +# --------------------------------------------------------------- vocabulary + +Remote = Literal["", "remote", "hybrid", "onsite"] +REMOTE: tuple[str, ...] = get_args(Remote) + +Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer", + "rejected", "withdrawn"] +STATUSES: tuple[str, ...] = get_args(Status) + +#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score). +AFFINITY = range(-1, 3) + + +# ------------------------------------------------------------------ models + +class Role(Model): + """A row of `roles`: one posting on one board.""" + + uid: str + source: str + source_id: str + subscription_id: int | None = None + company_key: str = "" + company: str = "" + title: str = "" + title_key: str = "" + place: str = "" + place_key: str = "" + country: str = "" + remote: Remote = "" + salary_min: int | None = None + salary_max: int | None = None + currency: str = "" + seniority: str = "" + function: str = "" + description: str = "" + feed_text: str = "" + original_url: str = "" + published_at: str + first_seen: str + last_seen: str + closed_at: str | None = None + raw: JsonDict = {} + extracted_at: str | None = None + created_at: str + + +class Company(Model): + key: str + name: str + domain: str = "" + affinity: int = 0 + status: str = "researching" + careers_url: str = "" + notes: str = "" + created_at: str + updated_at: str + + +class Subscription(Model): + """A row of `subscriptions`. `company` and `board` are the two labels + newsdesk calls outlet and section.""" + + id: int + source: str + target: str + label: str + company: str = "" + board: str = "" + params: JsonDict = {} + subscribed: int = 1 + fetch: int = 1 + show: int = 1 + added_at: str + target_id: str = "" + + +class Application(Model): + """A row of `applications`. `id` empty means not saved yet; the store mints + one. `role_uid` is None for a role that was never posted.""" + + id: str = "" + role_uid: str | None = None + company_key: str = "" + role_title: str = "" + url: str = "" + status: Status = "saved" + source: str = "" + applied_at: str | None = None + next_action: str = "" + next_date: str | None = None + contact: str = "" + notes: str = "" + created_at: str = "" + updated_at: str = "" + + +class ApplicationEvent(Model): + id: int + application_id: str + status: Status + at: str diff --git a/roledesk/storage/schema.py b/roledesk/storage/schema.py index 5c7ee3d..447008c 100644 --- a/roledesk/storage/schema.py +++ b/roledesk/storage/schema.py @@ -19,7 +19,142 @@ from collections.abc import Callable log = logging.getLogger(__name__) SCHEMA = """ --- The string map a desk plugin is handed as `self.state`. One namespace shared +-- One posting on one board. `uid` is sha1(source, source_id), the identity desk +-- gives every Item. The gate fields are real columns, unpacked from the item's +-- raw on upsert or filled later by extract: raw is transport, the columns are +-- what gets queried. +CREATE TABLE IF NOT EXISTS roles ( + uid TEXT PRIMARY KEY, + source TEXT NOT NULL, + source_id TEXT NOT NULL, + subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL, + -- The cluster key is (company_key, title_key, place_key). No foreign key + -- to companies: a role may arrive before anyone has made a company row. + company_key TEXT NOT NULL DEFAULT '', + company TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + title_key TEXT NOT NULL DEFAULT '', + -- Geography. place and not location, which is where an Item's original + -- lives. remote and country are separate so a gate on remote can also + -- require the country: "UK | Remote" is not remote from Berlin. + place TEXT NOT NULL DEFAULT '', + place_key TEXT NOT NULL DEFAULT '', + country TEXT NOT NULL DEFAULT '', + remote TEXT NOT NULL DEFAULT '' + CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')), + salary_min INTEGER, + salary_max INTEGER, + currency TEXT NOT NULL DEFAULT '', + seniority TEXT NOT NULL DEFAULT '', + function TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', -- the posting's body text + feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over + original_url TEXT NOT NULL DEFAULT '', -- the posting page + published_at TEXT NOT NULL, -- what the source claims, UTC + -- What storing postings buys. first_seen is the only honest posted date, + -- since Greenhouse resets updated_at on every edit. closed_at is set when a + -- board fetched in full no longer lists the role, and cleared if it returns. + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + closed_at TEXT, + raw TEXT NOT NULL DEFAULT '{}', + extracted_at TEXT, -- when extract last filled columns + created_at TEXT NOT NULL +); + +-- A company as a target, whether or not any board of it is read. key is +-- models.company_key(name), the slug roles.company_key holds. +CREATE TABLE IF NOT EXISTS companies ( + key TEXT PRIMARY KEY, + name TEXT NOT NULL, + domain TEXT NOT NULL DEFAULT '', + -- -1 mute, 0 neutral, 1 interested, 2 favourite. + affinity INTEGER NOT NULL DEFAULT 0 CHECK (affinity BETWEEN -1 AND 2), + status TEXT NOT NULL DEFAULT 'researching', + careers_url TEXT NOT NULL DEFAULT '', + notes TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +-- The targets desk's runner hands to plugins. newsdesk's shape, with its two +-- labels named company and board from the first migration. +CREATE TABLE IF NOT EXISTS subscriptions ( + id INTEGER PRIMARY KEY, + source TEXT NOT NULL, -- plugin name + target TEXT NOT NULL, -- e.g. greenhouse:acme + label TEXT NOT NULL, -- what Item.subscription names + company TEXT NOT NULL DEFAULT '', + board TEXT NOT NULL DEFAULT '', + params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with + subscribed INTEGER NOT NULL DEFAULT 1, + fetch INTEGER NOT NULL DEFAULT 1, + show INTEGER NOT NULL DEFAULT 1, + added_at TEXT NOT NULL, + target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it + UNIQUE (source, target), + UNIQUE (source, label) +); + +-- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that +-- was never posted, so an unsolicited approach is tracked like any other. +CREATE TABLE IF NOT EXISTS applications ( + id TEXT PRIMARY KEY, + role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL, + company_key TEXT NOT NULL DEFAULT '', + role_title TEXT NOT NULL DEFAULT '', + url TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'saved' + CHECK (status IN ('saved', 'applied', 'screen', 'technical', + 'onsite', 'offer', 'rejected', 'withdrawn')), + source TEXT NOT NULL DEFAULT '', + applied_at TEXT, + next_action TEXT NOT NULL DEFAULT '', + next_date TEXT, + contact TEXT NOT NULL DEFAULT '', + notes TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +-- One row per status an application entered, written by the store and never +-- updated: what days-to-first-screen and conversion by source are read from. +CREATE TABLE IF NOT EXISTS application_events ( + id INTEGER PRIMARY KEY, + application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE, + status TEXT NOT NULL, + at TEXT NOT NULL +); + +CREATE TRIGGER IF NOT EXISTS application_events_append_only +BEFORE UPDATE ON application_events +BEGIN + SELECT RAISE(ABORT, 'application_events is append-only'); +END; + +-- A search agent: a named saved search with its own intent text, gates and +-- weights. gates is a list of {field, values, mode: require|prefer, weight}. +CREATE TABLE IF NOT EXISTS agents ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + intent TEXT NOT NULL DEFAULT '', + gates TEXT NOT NULL DEFAULT '[]', + weights TEXT NOT NULL DEFAULT '{}', + enabled INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL +); + +-- A role is scored per agent. A gated role keeps its row, with the reason. +CREATE TABLE IF NOT EXISTS scores ( + uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, + agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + score REAL NOT NULL DEFAULT 0, + gated_reason TEXT NOT NULL DEFAULT '', + scored_at TEXT NOT NULL, + PRIMARY KEY (uid, agent_id) +); + +-- The string map a desk plugin is handed as self.state. One namespace shared -- by every plugin, so a plugin prefixes its keys. CREATE TABLE IF NOT EXISTS state ( key TEXT PRIMARY KEY, @@ -30,18 +165,27 @@ CREATE TABLE IF NOT EXISTS state ( #: Applied after `SCHEMA` or the migrations, every open. An index may name a #: column only a migration adds, which is why these are kept apart. INDEXES = """ +CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key); +CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key); +CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC); +CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at); +CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key); +CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id); +CREATE INDEX IF NOT EXISTS idx_application_events + ON application_events(application_id, at); +CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC); """ #: The table whose absence marks a file that has never been a roledesk #: database, and is therefore created from `SCHEMA` rather than migrated. -MARKER = "state" +MARKER = "roles" Step = str | Callable[[sqlite3.Connection], None] MIGRATIONS: tuple[tuple[int, str, Step], ...] = ( # Never run as a migration: a file without the marker table is created # from `SCHEMA` and stamped. It exists so the first real migration is 2. - (1, "the first schema", ""), + (1, "roles, companies, subscriptions, applications, agents, scores, state", ""), ) #: What a file created from `SCHEMA` today is already at. @@ -59,8 +203,9 @@ def _stamp(conn: sqlite3.Connection, version: int) -> None: def _statements(script: str) -> list[str]: """A script split into statements, cut where `sqlite3.complete_statement` - says one ends. Not `executescript`, which commits before it starts and - autocommits each statement, so a failing migration would be half applied.""" + says one ends, which knows about comments and trigger bodies. Not + `executescript`, which commits before it starts and autocommits each + statement, so a failing migration would be half applied.""" statements, buffer = [], "" for line in script.splitlines(keepends=True): buffer += line diff --git a/roledesk/storage/store.py b/roledesk/storage/store.py index 4ebb4ce..6ac659c 100644 --- a/roledesk/storage/store.py +++ b/roledesk/storage/store.py @@ -8,14 +8,57 @@ that costs a file handle. from __future__ import annotations +import json import sqlite3 -from collections.abc import Iterator, MutableMapping +from collections.abc import Callable, Iterator, MutableMapping, Sequence from contextlib import closing, contextmanager +from datetime import UTC, datetime from pathlib import Path from typing import Any +from uuid import uuid4 +from desk.models import Item +from roledesk.models import ( + AFFINITY, REMOTE, STATUSES, Application, ApplicationEvent, Company, Role, + Subscription, company_key, place_key, title_key, +) from roledesk.storage import schema +#: Text columns a plugin may fill from `Item.raw` under the same name. An +#: empty value never overwrites a filled column, so an upsert from a board that +#: says less than `extract` found keeps what `extract` found. +_RAW_TEXT = ("place", "country", "currency", "seniority", "function") +_RAW_INT = ("salary_min", "salary_max") + +_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes") +_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch", + "show", "target_id") + + +def _iso(moment: datetime) -> str: + """UTC, to the second. Every timestamp column is compared as text, so every + writer has to spell it the same way.""" + return moment.astimezone(UTC).isoformat(timespec="seconds") + + +def _remote(value: Any) -> str: + """A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything + unrecognised is '' (unknown), which `extract` may fill later.""" + if value is True: + return "remote" + text = str(value or "").strip().lower().replace("-", "").replace(" ", "") + text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text) + return text if text in REMOTE else "" + + +def _int_or_none(value: Any) -> int | None: + if value is None or value == "" or isinstance(value, bool): + return None + try: + return int(float(value)) + except (TypeError, ValueError): + return None + class _StateMap(MutableMapping[str, str]): """`Store.state`: the key/value table, read and written one key at a time.""" @@ -51,6 +94,8 @@ class Store: def __init__(self, path: str | Path) -> None: self.path = Path(path).expanduser() self.path.parent.mkdir(parents=True, exist_ok=True) + #: What "now" is. Replaced by tests that need two writes a second apart. + self.clock: Callable[[], datetime] = lambda: datetime.now(UTC) with self.transaction() as conn: # Persistent: recorded in the file, inherited by every connection. conn.execute("PRAGMA journal_mode=WAL") @@ -94,6 +139,292 @@ class Store: with self.transaction() as conn: return conn.execute(sql, params).rowcount + def now(self) -> str: + return _iso(self.clock()) + + # ---------------------------------------------------------------- roles + + def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool: + """Store one posting. Returns True when it was new. + + `first_seen` and `created_at` are written on insert only; `last_seen` on + every call, and a role seen again is open again. The gate columns come + from `item.raw` under their own names, with `company` falling back to + the subscription's; an empty value never overwrites a filled column. + """ + if not item.source: + raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") + raw = item.raw or {} + company = str(raw.get("company") or (subscription.company if subscription else "")) + place = str(raw.get("place") or "") + stamp = self.now() + row: dict[str, Any] = { + "uid": item.uid, + "source": item.source, + "source_id": item.source_id, + "subscription_id": subscription.id if subscription else None, + "company_key": company_key(company), + "company": company, + "title": item.title, + "title_key": title_key(item.title), + "place_key": place_key(place), + "remote": _remote(raw.get("remote")), + "description": item.body, + "feed_text": item.excerpt, + "original_url": item.location, + "published_at": _iso(item.published_at), + "first_seen": stamp, + "last_seen": stamp, + "raw": json.dumps(raw, default=str), + "created_at": stamp, + } + row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT} + row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} + + keep_filled = ("company", "company_key", "place_key", "remote", "description", + "feed_text", "original_url", *_RAW_TEXT) + updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled] + updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})" + for c in (*_RAW_INT, "subscription_id")] + updates += [f"{c}=excluded.{c}" + for c in ("title", "title_key", "published_at", "raw", "last_seen")] + updates.append("closed_at=NULL") + columns = ", ".join(row) + marks = ", ".join(f":{c}" for c in row) + with self.transaction() as conn: + existed = conn.execute("SELECT 1 FROM roles WHERE uid=?", + (row["uid"],)).fetchone() is not None + conn.execute( + f"INSERT INTO roles ({columns}) VALUES ({marks}) " + f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row) + return not existed + + def role(self, uid: str) -> Role | None: + row = self._row("SELECT * FROM roles WHERE uid=?", (uid,)) + return Role.from_row(row) if row else None + + def roles(self, limit: int = 100, company: str | None = None, + place: str | None = None, open_only: bool = True) -> list[Role]: + """Newest first by `first_seen`. `company` and `place` are matched by + key, so any spelling that keys the same finds the same rows.""" + where, params = [], [] + if company: + where.append("company_key = ?") + params.append(company_key(company)) + if place: + where.append("place_key = ?") + params.append(place_key(place)) + if open_only: + where.append("closed_at IS NULL") + sql = "SELECT * FROM roles" + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY first_seen DESC, uid LIMIT ?" + return [Role.from_row(r) for r in self._rows(sql, [*params, limit])] + + def mark_closed(self, source: str, seen_ids: Sequence[str], + subscription_id: int) -> int: + """Close every open role of one board that a complete fetch of it did not + list. Returns how many closed. Only call it after a fetch that finished: + a failed one lists nothing, and would close the whole board.""" + return self._write( + """UPDATE roles SET closed_at = ? + WHERE source = ? AND subscription_id = ? AND closed_at IS NULL + AND source_id NOT IN (SELECT value FROM json_each(?))""", + (self.now(), source, subscription_id, json.dumps(list(seen_ids)))) + + # ------------------------------------------------------------ companies + + def add_company(self, name: str, *, key: str = "", domain: str = "", + affinity: int = 0, status: str = "researching", + careers_url: str = "", notes: str = "") -> Company: + """Raises ValueError on a key already taken or an affinity out of range.""" + name = name.strip() + key = key or company_key(name) + if not key: + raise ValueError("a company needs a name") + _check_affinity(affinity) + stamp = self.now() + try: + self._write( + """INSERT INTO companies (key, name, domain, affinity, status, + careers_url, notes, created_at, updated_at) + VALUES (?,?,?,?,?,?,?,?,?)""", + (key, name, domain, affinity, status, careers_url, notes, stamp, stamp)) + except sqlite3.IntegrityError as exc: + raise ValueError(f"company {key!r} already exists") from exc + return self._company(key) + + def company(self, key: str) -> Company | None: + row = self._row("SELECT * FROM companies WHERE key=?", (key,)) + return Company.from_row(row) if row else None + + def companies(self) -> list[Company]: + return [Company.from_row(r) + for r in self._rows("SELECT * FROM companies ORDER BY name COLLATE NOCASE")] + + def update_company(self, key: str, **fields: Any) -> Company: + """Change any of name, domain, affinity, status, careers_url, notes.""" + unknown = set(fields) - set(_COMPANY_FIELDS) + if unknown: + raise ValueError(f"not a company field: {', '.join(sorted(unknown))}") + if "affinity" in fields: + _check_affinity(fields["affinity"]) + if fields: + sets = ", ".join(f"{f}=?" for f in fields) + changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?", + (*fields.values(), self.now(), key)) + if not changed: + raise KeyError(key) + return self._company(key) + + def remove_company(self, key: str) -> bool: + return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0 + + def _company(self, key: str) -> Company: + found = self.company(key) + if found is None: + raise KeyError(key) + return found + + # -------------------------------------------------------- subscriptions + + def subscriptions(self, source: str | None = None, + fetchable: bool = False) -> list[Subscription]: + """`fetchable` is what a pull reads: subscribed and with fetch on.""" + where, params = [], [] + if source: + where.append("source = ?") + params.append(source) + if fetchable: + where.append("fetch = 1 AND subscribed = 1") + sql = "SELECT * FROM subscriptions" + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY company, board, label" + return [Subscription.from_row(r) for r in self._rows(sql, params)] + + def subscription(self, sub_id: int) -> Subscription | None: + row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,)) + return Subscription.from_row(row) if row else None + + def subscription_for(self, source: str, label: str) -> Subscription | None: + """The row an item names: `Item.subscription` is the label.""" + row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?", + (source, label)) + return Subscription.from_row(row) if row else None + + def add_subscription(self, source: str, target: str, label: str = "", + company: str = "", board: str = "", + params: dict[str, Any] | None = None, + target_id: str = "") -> Subscription: + """Raises ValueError on a duplicate target or label within one plugin.""" + target = target.strip() + if not target: + raise ValueError("a subscription needs a target") + label = label.strip() or target + try: + with self.transaction() as conn: + new_id = conn.execute( + """INSERT INTO subscriptions (source, target, label, company, board, + params, added_at, target_id) + VALUES (?,?,?,?,?,?,?,?)""", + (source, target, label, company, board, json.dumps(params or {}), + self.now(), target_id)).lastrowid + except sqlite3.IntegrityError as exc: + raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc + assert new_id is not None + return self._subscription(new_id) + + def update_subscription(self, sub_id: int, **fields: Any) -> Subscription: + """Change labels, params, target_id or the three switches.""" + unknown = set(fields) - set(_SUBSCRIPTION_FIELDS) + if unknown: + raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}") + if "params" in fields: + fields["params"] = json.dumps(fields["params"] or {}) + for switch in ("subscribed", "fetch", "show"): + if switch in fields: + fields[switch] = int(bool(fields[switch])) + if fields: + sets = ", ".join(f"{f}=?" for f in fields) + try: + changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?", + (*fields.values(), sub_id)) + except sqlite3.IntegrityError as exc: + raise ValueError(f"label {fields.get('label')!r} is already in use") from exc + if not changed: + raise KeyError(sub_id) + return self._subscription(sub_id) + + def unsubscribe(self, sub_id: int) -> Subscription: + """Stop following it and hide it. The row stays, and so do its roles.""" + return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0) + + def _subscription(self, sub_id: int) -> Subscription: + found = self.subscription(sub_id) + if found is None: + raise KeyError(sub_id) + return found + + # --------------------------------------------------------- applications + + def save_application(self, application: Application) -> Application: + """Insert or update one application. An empty `id` is minted. Entering a + status, on insert or by change, appends an `application_events` row in + the same transaction.""" + _check_status(application.status) + app_id = application.id or uuid4().hex + stamp = self.now() + fields = application.model_dump(exclude={"id", "created_at", "updated_at"}) + with self.transaction() as conn: + before = conn.execute("SELECT status FROM applications WHERE id=?", + (app_id,)).fetchone() + if before is None: + columns = ["id", *fields, "created_at", "updated_at"] + conn.execute( + f"INSERT INTO applications ({', '.join(columns)}) " + f"VALUES ({', '.join('?' for _ in columns)})", + (app_id, *fields.values(), stamp, stamp)) + else: + sets = ", ".join(f"{f}=?" for f in fields) + conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?", + (*fields.values(), stamp, app_id)) + if before is None or before["status"] != application.status: + conn.execute("INSERT INTO application_events (application_id, status, at) " + "VALUES (?,?,?)", (app_id, application.status, stamp)) + return self._application(app_id) + + def set_application_status(self, app_id: str, status: str) -> Application: + """Move one application to `status`. Raises ValueError for a status + outside `STATUSES` and KeyError for an unknown id.""" + _check_status(status) + current = self._application(app_id) + return self.save_application(current.model_copy(update={"status": status})) + + def application(self, app_id: str) -> Application | None: + row = self._row("SELECT * FROM applications WHERE id=?", (app_id,)) + return Application.from_row(row) if row else None + + def applications(self, status: str | None = None) -> list[Application]: + sql, params = "SELECT * FROM applications", [] + if status: + sql += " WHERE status = ?" + params.append(status) + sql += " ORDER BY updated_at DESC, id" + return [Application.from_row(r) for r in self._rows(sql, params)] + + def application_events(self, app_id: str) -> list[ApplicationEvent]: + return [ApplicationEvent.from_row(r) for r in self._rows( + "SELECT * FROM application_events WHERE application_id=? ORDER BY id", + (app_id,))] + + def _application(self, app_id: str) -> Application: + found = self.application(app_id) + if found is None: + raise KeyError(app_id) + return found + # ---------------------------------------------------------------- state @property @@ -101,3 +432,14 @@ class Store: """The `state` table as a mapping. What the runner hands a plugin as `self.state`: it can keep a counter without being able to read roles.""" return _StateMap(self) + + +def _check_status(status: str) -> None: + if status not in STATUSES: + raise ValueError(f"{status!r} is not an application status " + f"({', '.join(STATUSES)})") + + +def _check_affinity(affinity: Any) -> None: + if not isinstance(affinity, int) or affinity not in AFFINITY: + raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}") diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..5e62210 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,248 @@ +"""The store: schema, role upserts, closure, companies, subscriptions, the tracker.""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from desk.models import Item +from roledesk.models import Application, company_key, place_key, title_key +from roledesk.storage import LATEST, Store + + +class Clock: + def __init__(self) -> None: + self.at = datetime(2026, 9, 23, 9, 0, tzinfo=UTC) + + def __call__(self) -> datetime: + return self.at + + def tick(self, hours: float = 1) -> None: + self.at += timedelta(hours=hours) + + +@pytest.fixture +def clock() -> Clock: + return Clock() + + +@pytest.fixture +def store(tmp_path: Path, clock: Clock) -> Iterator[Store]: + s = Store(tmp_path / "roledesk.db") + s.clock = clock + yield s + + +def posting(source_id: str = "123", **raw: object) -> Item: + item = Item(source_id=source_id, location=f"https://boards.example/acme/{source_id}", + title="Senior Backend Engineer (m/w/d)", body="We build things.", + published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw)) + item.source = "ats" + return item + + +# ------------------------------------------------------------------ schema + +def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: + path = tmp_path / "new.db" + store = Store(path) + with sqlite3.connect(path) as conn: + tables = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'")} + assert conn.execute("PRAGMA user_version").fetchone()[0] == LATEST + assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal" + indexes = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='index'")} + assert {"roles", "companies", "subscriptions", "applications", + "application_events", "agents", "scores", "state"} <= tables + assert "idx_roles_cluster" in indexes + assert store.migrations_applied == [] + # A second open finds it current and applies nothing. + assert Store(path).migrations_applied == [] + + +def test_a_newer_file_is_refused(tmp_path: Path) -> None: + path = tmp_path / "future.db" + Store(path) + with sqlite3.connect(path) as conn: + conn.execute(f"PRAGMA user_version = {LATEST + 1}") + with pytest.raises(RuntimeError, match=r"Upgrade roledesk"): + Store(path) + + +def test_state_is_a_mapping(store: Store) -> None: + store.state["ats:last"] = "2026-09-23" + assert store.state["ats:last"] == "2026-09-23" + assert dict(store.state) == {"ats:last": "2026-09-23"} + del store.state["ats:last"] + assert "ats:last" not in store.state + + +# ------------------------------------------------------------------- keys + +def test_keys() -> None: + assert title_key("Senior Backend Engineer (m/w/d)") == "senior backend engineer" + assert title_key("Data Scientist (f/m/x) – Berlin") == "data scientist berlin" # noqa: RUF001 + assert title_key("Product Designer m/w/d") == "product designer" + assert title_key("Engineer (all genders)") == "engineer" + assert place_key("Zürich, Switzerland") == "zurich switzerland" + assert company_key("Trade Republic Bank GmbH") == "trade-republic-bank" + assert company_key("Celonis SE") == company_key("celonis") == "celonis" + + +# ------------------------------------------------------------------- roles + +def test_upsert_twice_keeps_first_seen_and_bumps_last_seen( + store: Store, clock: Clock) -> None: + assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is True + first = store.role(posting().uid) + assert first is not None + clock.tick() + assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is False + again = store.role(posting().uid) + assert again is not None + assert again.first_seen == first.first_seen == first.created_at + assert again.last_seen > first.last_seen + assert (again.company_key, again.title_key, again.place_key) == \ + ("acme", "senior backend engineer", "berlin") + + +def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: + store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000", + salary_max=95000, currency="EUR", department="Eng")) + role = store.role(posting().uid) + assert role is not None + assert (role.remote, role.salary_min, role.salary_max, role.currency) == \ + ("remote", 80000, 95000, "EUR") + assert role.raw["department"] == "Eng" + assert role.description == "We build things." + assert role.original_url.endswith("/123") + # A later fetch saying less does not erase what was known. + store.upsert_role(posting()) + role = store.role(posting().uid) + assert role is not None + assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000) + + +def test_company_falls_back_to_the_subscription(store: Store) -> None: + sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse") + store.upsert_role(posting(), sub) + role = store.role(posting().uid) + assert role is not None + assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id) + + +def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None: + store.upsert_role(posting("1", company="Acme", place="Berlin")) + clock.tick() + store.upsert_role(posting("2", company="Acme", place="Munich")) + clock.tick() + store.upsert_role(posting("3", company="Other", place="Berlin")) + assert [r.source_id for r in store.roles()] == ["3", "2", "1"] + assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"] + assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"] + assert [r.source_id for r in store.roles(limit=1)] == ["3"] + + +def test_mark_closed_closes_what_the_board_no_longer_lists(store: Store) -> None: + sub = store.add_subscription("ats", "greenhouse:acme") + other = store.add_subscription("ats", "greenhouse:other") + for source_id in ("1", "2", "3"): + store.upsert_role(posting(source_id), sub) + store.upsert_role(posting("9"), other) + assert store.mark_closed("ats", ["1", "3"], sub.id) == 1 + assert {r.source_id for r in store.roles()} == {"1", "3", "9"} + assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"} + # Listed again: open again. + store.upsert_role(posting("2"), sub) + assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"} + + +def test_an_item_without_a_source_is_refused(store: Store) -> None: + item = posting() + item.source = "" + with pytest.raises(ValueError, match="no source"): + store.upsert_role(item) + + +# --------------------------------------------------------------- companies + +def test_companies_crud(store: Store) -> None: + acme = store.add_company("Acme GmbH", domain="acme.example") + assert acme.key == "acme" and acme.affinity == 0 and acme.status == "researching" + with pytest.raises(ValueError, match="already exists"): + store.add_company("Acme") + updated = store.update_company("acme", affinity=2, status="contacted") + assert (updated.affinity, updated.status) == (2, "contacted") + with pytest.raises(ValueError, match="affinity"): + store.update_company("acme", affinity=5) + with pytest.raises(ValueError, match="not a company field"): + store.update_company("acme", created_at="x") + assert [c.key for c in store.companies()] == ["acme"] + assert store.remove_company("acme") is True + assert store.company("acme") is None + + +# ----------------------------------------------------------- subscriptions + +def test_subscriptions_crud(store: Store) -> None: + sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", + company="Acme", board="Greenhouse", params={"q": 1}) + assert (sub.company, sub.board, sub.params, sub.fetch) == \ + ("Acme", "Greenhouse", {"q": 1}, 1) + with pytest.raises(ValueError): + store.add_subscription("ats", "greenhouse:acme") + assert store.subscription_for("ats", "Acme") == sub + store.update_subscription(sub.id, fetch=False) + assert store.subscriptions(fetchable=True) == [] + assert [s.id for s in store.subscriptions(source="ats")] == [sub.id] + gone = store.unsubscribe(sub.id) + assert (gone.subscribed, gone.fetch, gone.show) == (0, 0, 0) + + +# ------------------------------------------------------------ applications + +def test_a_status_change_writes_an_event(store: Store, clock: Clock) -> None: + saved = store.save_application(Application(company_key="acme", role_title="Engineer")) + assert saved.id and saved.status == "saved" + clock.tick() + store.set_application_status(saved.id, "applied") + clock.tick() + store.save_application(saved.model_copy(update={"status": "applied", "notes": "x"})) + clock.tick() + store.set_application_status(saved.id, "screen") + events = store.application_events(saved.id) + assert [e.status for e in events] == ["saved", "applied", "screen"] + assert events[0].at < events[1].at < events[2].at + assert [a.notes for a in store.applications()] == ["x"] + + +def test_an_invalid_status_is_refused(store: Store) -> None: + saved = store.save_application(Application(role_title="Engineer")) + with pytest.raises(ValueError, match="not an application status"): + store.set_application_status(saved.id, "ghosted") + with pytest.raises(ValidationError): + Application(status="ghosted") + with pytest.raises(sqlite3.IntegrityError), store.transaction() as conn: + conn.execute("UPDATE applications SET status='ghosted' WHERE id=?", (saved.id,)) + assert [e.status for e in store.application_events(saved.id)] == ["saved"] + + +def test_events_are_append_only(store: Store) -> None: + saved = store.save_application(Application(role_title="Engineer")) + with pytest.raises(sqlite3.IntegrityError, match="append-only"), \ + store.transaction() as conn: + conn.execute("UPDATE application_events SET status='offer'") + assert store.application_events(saved.id)[0].status == "saved" + + +def test_an_application_may_point_at_a_role(store: Store) -> None: + store.upsert_role(posting()) + saved = store.save_application(Application(role_uid=posting().uid, status="applied")) + assert saved.role_uid == posting().uid + assert [e.status for e in store.application_events(saved.id)] == ["applied"]