Companies as symbols: company_aliases, merge, duplicates, empty enrichment columns (migration 20)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:45:43 +02:00
1 parent 3adafd8950
commit 9172b2da72
8 files changed
+347 -17

No files matched your search

+59 -2
View File
@@ -18,7 +18,7 @@ import sqlite3
from collections.abc import Callable
from app import places
from app.models import SENIORITIES, country_of, posted_at, posted_host
from app.models import SENIORITIES, company_fold, country_of, posted_at, posted_host
from app.seniority import from_title
from app.storage import gazetteer
@@ -198,9 +198,29 @@ CREATE TABLE IF NOT EXISTS companies (
ats_slug TEXT NOT NULL DEFAULT '',
watch INTEGER CHECK (watch IN (0, 1)),
careers_opened_at TEXT NOT NULL DEFAULT '',
hand_reason TEXT NOT NULL DEFAULT ''
hand_reason TEXT NOT NULL DEFAULT '',
sector TEXT NOT NULL DEFAULT '',
stage TEXT NOT NULL DEFAULT '',
size_band TEXT NOT NULL DEFAULT '',
founded INTEGER,
last_funding INTEGER,
last_funding_currency TEXT NOT NULL DEFAULT '',
facts_source TEXT NOT NULL DEFAULT '',
facts_at TEXT NOT NULL DEFAULT ''
);
-- Every folded spelling of a company (models.company_fold): orsted is Ørsted
-- whichever way a board writes it, and a person can add another name
-- ("Dong Energy"). One fold may name several companies until a person merges
-- them (Store.merge_companies). The enrichment columns above are empty until
-- a reader states them; facts_source and facts_at say where and when.
CREATE TABLE IF NOT EXISTS company_aliases (
fold TEXT NOT NULL,
company_key TEXT NOT NULL,
UNIQUE (fold, company_key)
);
CREATE INDEX IF NOT EXISTS idx_company_aliases_key ON company_aliases(company_key);
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
-- labels named company and board from the first migration.
CREATE TABLE IF NOT EXISTS subscriptions (
@@ -536,6 +556,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
"places, resolved against a gazetteer; roles.teaser; the three seniority "
"sources", lambda conn: _add_places(conn)),
# A Python step: the seed folds every name.
(20, "company_aliases and the empty enrichment columns of companies",
lambda conn: _add_company_aliases(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -734,6 +757,40 @@ def _add_places(conn: sqlite3.Connection) -> None:
len(gaz.places), written, len(by_uid), len(levels))
_V20 = """
ALTER TABLE companies ADD COLUMN sector TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN stage TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN size_band TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN founded INTEGER;
ALTER TABLE companies ADD COLUMN last_funding INTEGER;
ALTER TABLE companies ADD COLUMN last_funding_currency TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN facts_source TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN facts_at TEXT NOT NULL DEFAULT '';
CREATE TABLE company_aliases (
fold TEXT NOT NULL,
company_key TEXT NOT NULL,
UNIQUE (fold, company_key)
);
CREATE INDEX idx_company_aliases_key ON company_aliases(company_key);
"""
def _add_company_aliases(conn: sqlite3.Connection) -> None:
"""Migration 20's step. The columns and the table, and every company's
name folded into it. Companies whose folds collide are logged and left
alone: merging is a person's call (`Store.merge_companies`)."""
for statement in _statements(_V20):
conn.execute(statement)
rows = conn.execute("SELECT key, name FROM companies").fetchall()
conn.executemany("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)",
[(fold, key) for key, name in rows if (fold := company_fold(name))])
clashes = conn.execute("SELECT fold, group_concat(company_key, ', ') FROM company_aliases "
"GROUP BY fold HAVING COUNT(*) > 1").fetchall()
for fold, keys in clashes:
log.warning("migration 20: %r is one fold of %s", fold, keys)
log.info("migration 20: %d companies folded, %d folds shared", len(rows), len(clashes))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])