Companies as symbols: company_aliases, merge, duplicates, empty enrichment columns (migration 20)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3adafd8950
commit
9172b2da72
8 files changed
+347
-17
No files matched your search
+59
-2
@@ -18,7 +18,7 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import SENIORITIES, country_of, posted_at, posted_host
|
||||
from app.models import SENIORITIES, company_fold, country_of, posted_at, posted_host
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
@@ -198,9 +198,29 @@ CREATE TABLE IF NOT EXISTS companies (
|
||||
ats_slug TEXT NOT NULL DEFAULT '',
|
||||
watch INTEGER CHECK (watch IN (0, 1)),
|
||||
careers_opened_at TEXT NOT NULL DEFAULT '',
|
||||
hand_reason TEXT NOT NULL DEFAULT ''
|
||||
hand_reason TEXT NOT NULL DEFAULT '',
|
||||
sector TEXT NOT NULL DEFAULT '',
|
||||
stage TEXT NOT NULL DEFAULT '',
|
||||
size_band TEXT NOT NULL DEFAULT '',
|
||||
founded INTEGER,
|
||||
last_funding INTEGER,
|
||||
last_funding_currency TEXT NOT NULL DEFAULT '',
|
||||
facts_source TEXT NOT NULL DEFAULT '',
|
||||
facts_at TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- Every folded spelling of a company (models.company_fold): orsted is Ørsted
|
||||
-- whichever way a board writes it, and a person can add another name
|
||||
-- ("Dong Energy"). One fold may name several companies until a person merges
|
||||
-- them (Store.merge_companies). The enrichment columns above are empty until
|
||||
-- a reader states them; facts_source and facts_at say where and when.
|
||||
CREATE TABLE IF NOT EXISTS company_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
company_key TEXT NOT NULL,
|
||||
UNIQUE (fold, company_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_company_aliases_key ON company_aliases(company_key);
|
||||
|
||||
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
|
||||
-- labels named company and board from the first migration.
|
||||
CREATE TABLE IF NOT EXISTS subscriptions (
|
||||
@@ -536,6 +556,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
|
||||
"places, resolved against a gazetteer; roles.teaser; the three seniority "
|
||||
"sources", lambda conn: _add_places(conn)),
|
||||
# A Python step: the seed folds every name.
|
||||
(20, "company_aliases and the empty enrichment columns of companies",
|
||||
lambda conn: _add_company_aliases(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -734,6 +757,40 @@ def _add_places(conn: sqlite3.Connection) -> None:
|
||||
len(gaz.places), written, len(by_uid), len(levels))
|
||||
|
||||
|
||||
_V20 = """
|
||||
ALTER TABLE companies ADD COLUMN sector TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN stage TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN size_band TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN founded INTEGER;
|
||||
ALTER TABLE companies ADD COLUMN last_funding INTEGER;
|
||||
ALTER TABLE companies ADD COLUMN last_funding_currency TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN facts_source TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE companies ADD COLUMN facts_at TEXT NOT NULL DEFAULT '';
|
||||
CREATE TABLE company_aliases (
|
||||
fold TEXT NOT NULL,
|
||||
company_key TEXT NOT NULL,
|
||||
UNIQUE (fold, company_key)
|
||||
);
|
||||
CREATE INDEX idx_company_aliases_key ON company_aliases(company_key);
|
||||
"""
|
||||
|
||||
|
||||
def _add_company_aliases(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 20's step. The columns and the table, and every company's
|
||||
name folded into it. Companies whose folds collide are logged and left
|
||||
alone: merging is a person's call (`Store.merge_companies`)."""
|
||||
for statement in _statements(_V20):
|
||||
conn.execute(statement)
|
||||
rows = conn.execute("SELECT key, name FROM companies").fetchall()
|
||||
conn.executemany("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)",
|
||||
[(fold, key) for key, name in rows if (fold := company_fold(name))])
|
||||
clashes = conn.execute("SELECT fold, group_concat(company_key, ', ') FROM company_aliases "
|
||||
"GROUP BY fold HAVING COUNT(*) > 1").fetchall()
|
||||
for fold, keys in clashes:
|
||||
log.warning("migration 20: %r is one fold of %s", fold, keys)
|
||||
log.info("migration 20: %d companies folded, %d folds shared", len(rows), len(clashes))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
Reference in new issue
Block a user