Merge branch 'wp-h-companies'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
06b464790a
8 files changed
+347
-17
No files matched your search
+59
-2
@@ -18,7 +18,7 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import SENIORITIES, country_of, posted_at, posted_host
|
||||
from app.models import SENIORITIES, company_fold, country_of, posted_at, posted_host
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
@@ -198,9 +198,29 @@ CREATE TABLE IF NOT EXISTS companies (
|
||||
ats_slug TEXT NOT NULL DEFAULT '',
|
||||
watch INTEGER CHECK (watch IN (0, 1)),
|
||||
careers_opened_at TEXT NOT NULL DEFAULT '',
|
||||
hand_reason TEXT NOT NULL DEFAULT ''
|
||||
hand_reason TEXT NOT NULL DEFAULT '',
|
||||
sector TEXT NOT NULL DEFAULT '',
|
||||
stage TEXT NOT NULL DEFAULT '',
|
||||
size_band TEXT NOT NULL DEFAULT '',
|
||||
founded INTEGER,
|
||||
last_funding INTEGER,
|
||||
last_funding_currency TEXT NOT NULL DEFAULT '',
|
||||
facts_source TEXT NOT NULL DEFAULT '',
|
||||
facts_at TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- Every folded spelling of a company (models.company_fold): orsted is Ørsted
|
||||
-- whichever way a board writes it, and a person can add another name
|
||||
-- ("Dong Energy"). One fold may name several companies until a person merges
|
||||
-- them (Store.merge_companies). The enrichment columns above are empty until
|
||||
-- a reader states them; facts_source and facts_at say where and when.
|
||||
CREATE TABLE IF NOT EXISTS company_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
company_key TEXT NOT NULL,
|
||||
UNIQUE (fold, company_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_company_aliases_key ON company_aliases(company_key);
|
||||
|
||||
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
|
||||
-- labels named company and board from the first migration.
|
||||
CREATE TABLE IF NOT EXISTS subscriptions (
|
||||
@@ -536,6 +556,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
|
||||
"places, resolved against a gazetteer; roles.teaser; the three seniority "
|
||||
"sources", lambda conn: _add_places(conn)),
|
||||
# A Python step: the seed folds every name.
|
||||
(20, "company_aliases and the empty enrichment columns of companies",
|
||||
lambda conn: _add_company_aliases(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -734,6 +757,40 @@ def _add_places(conn: sqlite3.Connection) -> None:
|
||||
len(gaz.places), written, len(by_uid), len(levels))
|
||||
|
||||
|
||||
_V20 = """
|
||||
ALTER TABLE companies ADD COLUMN sector TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN stage TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN size_band TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN founded INTEGER;
|
||||
ALTER TABLE companies ADD COLUMN last_funding INTEGER;
|
||||
ALTER TABLE companies ADD COLUMN last_funding_currency TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN facts_source TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN facts_at TEXT NOT NULL DEFAULT '';
|
||||
CREATE TABLE company_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
company_key TEXT NOT NULL,
|
||||
UNIQUE (fold, company_key)
|
||||
);
|
||||
CREATE INDEX idx_company_aliases_key ON company_aliases(company_key);
|
||||
"""
|
||||
|
||||
|
||||
def _add_company_aliases(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 20's step. The columns and the table, and every company's
|
||||
name folded into it. Companies whose folds collide are logged and left
|
||||
alone: merging is a person's call (`Store.merge_companies`)."""
|
||||
for statement in _statements(_V20):
|
||||
conn.execute(statement)
|
||||
rows = conn.execute("SELECT key, name FROM companies").fetchall()
|
||||
conn.executemany("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)",
|
||||
[(fold, key) for key, name in rows if (fold := company_fold(name))])
|
||||
clashes = conn.execute("SELECT fold, group_concat(company_key, ', ') FROM company_aliases "
|
||||
"GROUP BY fold HAVING COUNT(*) > 1").fetchall()
|
||||
for fold, keys in clashes:
|
||||
log.warning("migration 20: %r is one fold of %s", fold, keys)
|
||||
log.info("migration 20: %d companies folded, %d folds shared", len(rows), len(clashes))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+87
-7
@@ -25,7 +25,7 @@ from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
||||
RoleRow, ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
|
||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
@@ -114,7 +114,9 @@ _EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description
|
||||
_EXCERPT_RAW = 240
|
||||
|
||||
_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city",
|
||||
"ats_slug", "watch", "careers_opened_at", "hand_reason")
|
||||
"ats_slug", "watch", "careers_opened_at", "hand_reason", "sector", "stage",
|
||||
"size_band", "founded", "last_funding", "last_funding_currency",
|
||||
"facts_source", "facts_at")
|
||||
|
||||
#: A company as the store serves it: the row and the asked-for agent's affinity.
|
||||
_COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c
|
||||
@@ -610,7 +612,7 @@ class Store:
|
||||
"source": item.source,
|
||||
"source_id": item.source_id,
|
||||
"subscription_id": subscription.id if subscription else None,
|
||||
"company_key": company_key(company),
|
||||
"company_key": self.resolve_company(company),
|
||||
"company": company,
|
||||
"title": item.title,
|
||||
"title_key": title_key(item.title),
|
||||
@@ -1660,6 +1662,7 @@ class Store:
|
||||
f"INSERT INTO companies ({', '.join(columns)}) "
|
||||
f"VALUES ({', '.join('?' * len(columns))})",
|
||||
(key, name, *fields.values(), stamp, stamp))
|
||||
self._alias(conn, key, name)
|
||||
|
||||
def company(self, key: str, agent_id: int | None = None) -> Company | None:
|
||||
"""One company, with `affinity` as `agent_id` set it; 0 without an agent."""
|
||||
@@ -1715,10 +1718,13 @@ class Store:
|
||||
_check_hand_reason(fields["hand_reason"])
|
||||
if fields:
|
||||
sets = ", ".join(f"{f}=?" for f in fields)
|
||||
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
||||
(*fields.values(), self.now(), key))
|
||||
if not changed:
|
||||
raise KeyError(key)
|
||||
with self.transaction() as conn:
|
||||
changed = conn.execute(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
||||
(*fields.values(), self.now(), key)).rowcount
|
||||
if not changed:
|
||||
raise KeyError(key)
|
||||
if "name" in fields:
|
||||
self._alias(conn, key, fields["name"])
|
||||
return self._company(key)
|
||||
|
||||
def remove_company(self, key: str) -> bool:
|
||||
@@ -1728,8 +1734,82 @@ class Store:
|
||||
an affinity it never got. Roles and subscriptions are not touched."""
|
||||
with self.transaction() as conn:
|
||||
conn.execute("DELETE FROM agent_companies WHERE company_key=?", (key,))
|
||||
conn.execute("DELETE FROM company_aliases WHERE company_key=?", (key,))
|
||||
return conn.execute("DELETE FROM companies WHERE key=?", (key,)).rowcount > 0
|
||||
|
||||
# ---------------------------------------------------- company aliases
|
||||
|
||||
@staticmethod
|
||||
def _alias(conn: sqlite3.Connection, key: str, name: str) -> None:
|
||||
if fold := company_fold(name):
|
||||
conn.execute("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)",
|
||||
(fold, key))
|
||||
|
||||
def resolve_company(self, name: str) -> str:
|
||||
"""The key a role from `name` belongs to: the company an alias names,
|
||||
the one with the name's own key first when several share the fold,
|
||||
else `company_key(name)`. `Orsted` lands on the `ørsted` company."""
|
||||
own = company_key(name)
|
||||
row = self._row("SELECT company_key FROM company_aliases WHERE fold=? "
|
||||
"ORDER BY company_key=? DESC, company_key LIMIT 1",
|
||||
(company_fold(name), own))
|
||||
return str(row["company_key"]) if row else own
|
||||
|
||||
def add_alias(self, key: str, name: str) -> None:
|
||||
"""Another name for company `key`: a search and a role from `name` find
|
||||
it. Raises KeyError for an unknown company, ValueError for a name that
|
||||
folds to nothing."""
|
||||
if not company_fold(name):
|
||||
raise ValueError("an alias needs a name")
|
||||
with self.transaction() as conn:
|
||||
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
|
||||
raise KeyError(key)
|
||||
self._alias(conn, key, name)
|
||||
|
||||
def company_aliases(self) -> dict[str, list[str]]:
|
||||
"""`company_key -> folds`, for the company search."""
|
||||
out: dict[str, list[str]] = {}
|
||||
for r in self._rows("SELECT company_key, fold FROM company_aliases ORDER BY fold"):
|
||||
out.setdefault(str(r["company_key"]), []).append(str(r["fold"]))
|
||||
return out
|
||||
|
||||
def duplicate_companies(self) -> list[tuple[str, list[str]]]:
|
||||
"""`(fold, keys)` for each fold more than one company has, likely the
|
||||
same company under two spellings. Nothing is merged."""
|
||||
return [(str(r["fold"]), str(r["keys"]).split("\t")) for r in self._rows(
|
||||
"SELECT fold, group_concat(company_key, char(9)) AS keys FROM company_aliases "
|
||||
"GROUP BY fold HAVING COUNT(*) > 1 ORDER BY fold")]
|
||||
|
||||
def merge_companies(self, keep: str, other: str) -> Company:
|
||||
"""Fold company `other` into `keep`, in one transaction: its roles and
|
||||
subscriptions move to `keep`, each agent's stance moves unless the
|
||||
agent has one on `keep` already, its aliases and its name become
|
||||
aliases of `keep`, and its row goes. Copies are not recomputed
|
||||
(`rebuild_dupes`). Raises KeyError for an unknown company and
|
||||
ValueError for merging one into itself."""
|
||||
if keep == other:
|
||||
raise ValueError("a company cannot be merged into itself")
|
||||
with self.transaction() as conn:
|
||||
rows = {r["key"]: r["name"] for r in conn.execute(
|
||||
"SELECT key, name FROM companies WHERE key IN (?, ?)", (keep, other))}
|
||||
for key in (keep, other):
|
||||
if key not in rows:
|
||||
raise KeyError(key)
|
||||
conn.execute("UPDATE roles SET company_key=? WHERE company_key=?", (keep, other))
|
||||
for sub in conn.execute("SELECT id, company FROM subscriptions").fetchall():
|
||||
if company_key(sub["company"]) == other:
|
||||
conn.execute("UPDATE subscriptions SET company=? WHERE id=?",
|
||||
(rows[keep], sub["id"]))
|
||||
conn.execute("UPDATE OR IGNORE agent_companies SET company_key=? WHERE company_key=?",
|
||||
(keep, other))
|
||||
conn.execute("DELETE FROM agent_companies WHERE company_key=?", (other,))
|
||||
conn.execute("UPDATE OR IGNORE company_aliases SET company_key=? WHERE company_key=?",
|
||||
(keep, other))
|
||||
conn.execute("DELETE FROM company_aliases WHERE company_key=?", (other,))
|
||||
self._alias(conn, keep, rows[other])
|
||||
conn.execute("DELETE FROM companies WHERE key=?", (other,))
|
||||
return self._company(keep)
|
||||
|
||||
def _company(self, key: str, agent_id: int | None = None) -> Company:
|
||||
found = self.company(key, agent_id)
|
||||
if found is None:
|
||||
|
||||
Reference in new issue
Block a user