Companies as symbols: company_aliases, merge, duplicates, empty enrichment columns (migration 20)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:45:43 +02:00
1 parent 3adafd8950
commit 9172b2da72
8 files changed
+347 -17

No files matched your search

+7
View File
@@ -67,6 +67,13 @@ def company_key(company: str) -> str:
return _fold(name).replace(" ", "-")
def company_fold(company: str) -> str:
"""What `company_aliases` keys on: `company_key`'s cut of the legal form,
then the place fold, so "Orsted A/S" and "Ørsted" are both `orsted`. The
key itself keeps `ø`, since it is a primary key."""
return _place_fold(_LEGAL.sub("", (company or "").strip()))
# ---------------------------------------------------------------- geography
#: Letters NFKD leaves whole because they are letters of their own rather
+59 -2
View File
@@ -18,7 +18,7 @@ import sqlite3
from collections.abc import Callable
from app import places
from app.models import SENIORITIES, country_of, posted_at, posted_host
from app.models import SENIORITIES, company_fold, country_of, posted_at, posted_host
from app.seniority import from_title
from app.storage import gazetteer
@@ -198,9 +198,29 @@ CREATE TABLE IF NOT EXISTS companies (
ats_slug TEXT NOT NULL DEFAULT '',
watch INTEGER CHECK (watch IN (0, 1)),
careers_opened_at TEXT NOT NULL DEFAULT '',
hand_reason TEXT NOT NULL DEFAULT ''
hand_reason TEXT NOT NULL DEFAULT '',
sector TEXT NOT NULL DEFAULT '',
stage TEXT NOT NULL DEFAULT '',
size_band TEXT NOT NULL DEFAULT '',
founded INTEGER,
last_funding INTEGER,
last_funding_currency TEXT NOT NULL DEFAULT '',
facts_source TEXT NOT NULL DEFAULT '',
facts_at TEXT NOT NULL DEFAULT ''
);
-- Every folded spelling of a company (models.company_fold): orsted is Ørsted
-- whichever way a board writes it, and a person can add another name
-- ("Dong Energy"). One fold may name several companies until a person merges
-- them (Store.merge_companies). The enrichment columns above are empty until
-- a reader states them; facts_source and facts_at say where and when.
CREATE TABLE IF NOT EXISTS company_aliases (
fold TEXT NOT NULL,
company_key TEXT NOT NULL,
UNIQUE (fold, company_key)
);
CREATE INDEX IF NOT EXISTS idx_company_aliases_key ON company_aliases(company_key);
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
-- labels named company and board from the first migration.
CREATE TABLE IF NOT EXISTS subscriptions (
@@ -536,6 +556,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
"places, resolved against a gazetteer; roles.teaser; the three seniority "
"sources", lambda conn: _add_places(conn)),
# A Python step: the seed folds every name.
(20, "company_aliases and the empty enrichment columns of companies",
lambda conn: _add_company_aliases(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -734,6 +757,40 @@ def _add_places(conn: sqlite3.Connection) -> None:
len(gaz.places), written, len(by_uid), len(levels))
_V20 = """
ALTER TABLE companies ADD COLUMN sector TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN stage TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN size_band TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN founded INTEGER;
ALTER TABLE companies ADD COLUMN last_funding INTEGER;
ALTER TABLE companies ADD COLUMN last_funding_currency TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN facts_source TEXT NOT NULL DEFAULT '';
ALTER TABLE companies ADD COLUMN facts_at TEXT NOT NULL DEFAULT '';
CREATE TABLE company_aliases (
fold TEXT NOT NULL,
company_key TEXT NOT NULL,
UNIQUE (fold, company_key)
);
CREATE INDEX idx_company_aliases_key ON company_aliases(company_key);
"""
def _add_company_aliases(conn: sqlite3.Connection) -> None:
"""Migration 20's step. The columns and the table, and every company's
name folded into it. Companies whose folds collide are logged and left
alone: merging is a person's call (`Store.merge_companies`)."""
for statement in _statements(_V20):
conn.execute(statement)
rows = conn.execute("SELECT key, name FROM companies").fetchall()
conn.executemany("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)",
[(fold, key) for key, name in rows if (fold := company_fold(name))])
clashes = conn.execute("SELECT fold, group_concat(company_key, ', ') FROM company_aliases "
"GROUP BY fold HAVING COUNT(*) > 1").fetchall()
for fold, keys in clashes:
log.warning("migration 20: %r is one fold of %s", fold, keys)
log.info("migration 20: %d companies folded, %d folds shared", len(rows), len(clashes))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
+87 -7
View File
@@ -25,7 +25,7 @@ from app.models import (
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key,
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
)
from app.seniority import from_title
@@ -110,7 +110,9 @@ _EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description
_EXCERPT_RAW = 240
_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city",
"ats_slug", "watch", "careers_opened_at", "hand_reason")
"ats_slug", "watch", "careers_opened_at", "hand_reason", "sector", "stage",
"size_band", "founded", "last_funding", "last_funding_currency",
"facts_source", "facts_at")
#: A company as the store serves it: the row and the asked-for agent's affinity.
_COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c
@@ -606,7 +608,7 @@ class Store:
"source": item.source,
"source_id": item.source_id,
"subscription_id": subscription.id if subscription else None,
"company_key": company_key(company),
"company_key": self.resolve_company(company),
"company": company,
"title": item.title,
"title_key": title_key(item.title),
@@ -1625,6 +1627,7 @@ class Store:
f"INSERT INTO companies ({', '.join(columns)}) "
f"VALUES ({', '.join('?' * len(columns))})",
(key, name, *fields.values(), stamp, stamp))
self._alias(conn, key, name)
def company(self, key: str, agent_id: int | None = None) -> Company | None:
"""One company, with `affinity` as `agent_id` set it; 0 without an agent."""
@@ -1680,10 +1683,13 @@ class Store:
_check_hand_reason(fields["hand_reason"])
if fields:
sets = ", ".join(f"{f}=?" for f in fields)
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
(*fields.values(), self.now(), key))
if not changed:
raise KeyError(key)
with self.transaction() as conn:
changed = conn.execute(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
(*fields.values(), self.now(), key)).rowcount
if not changed:
raise KeyError(key)
if "name" in fields:
self._alias(conn, key, fields["name"])
return self._company(key)
def remove_company(self, key: str) -> bool:
@@ -1693,8 +1699,82 @@ class Store:
an affinity it never got. Roles and subscriptions are not touched."""
with self.transaction() as conn:
conn.execute("DELETE FROM agent_companies WHERE company_key=?", (key,))
conn.execute("DELETE FROM company_aliases WHERE company_key=?", (key,))
return conn.execute("DELETE FROM companies WHERE key=?", (key,)).rowcount > 0
# ---------------------------------------------------- company aliases
@staticmethod
def _alias(conn: sqlite3.Connection, key: str, name: str) -> None:
if fold := company_fold(name):
conn.execute("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)",
(fold, key))
def resolve_company(self, name: str) -> str:
"""The key a role from `name` belongs to: the company an alias names,
the one with the name's own key first when several share the fold,
else `company_key(name)`. `Orsted` lands on the `ørsted` company."""
own = company_key(name)
row = self._row("SELECT company_key FROM company_aliases WHERE fold=? "
"ORDER BY company_key=? DESC, company_key LIMIT 1",
(company_fold(name), own))
return str(row["company_key"]) if row else own
def add_alias(self, key: str, name: str) -> None:
"""Another name for company `key`: a search and a role from `name` find
it. Raises KeyError for an unknown company, ValueError for a name that
folds to nothing."""
if not company_fold(name):
raise ValueError("an alias needs a name")
with self.transaction() as conn:
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
raise KeyError(key)
self._alias(conn, key, name)
def company_aliases(self) -> dict[str, list[str]]:
"""`company_key -> folds`, for the company search."""
out: dict[str, list[str]] = {}
for r in self._rows("SELECT company_key, fold FROM company_aliases ORDER BY fold"):
out.setdefault(str(r["company_key"]), []).append(str(r["fold"]))
return out
def duplicate_companies(self) -> list[tuple[str, list[str]]]:
"""`(fold, keys)` for each fold more than one company has, likely the
same company under two spellings. Nothing is merged."""
return [(str(r["fold"]), str(r["keys"]).split("\t")) for r in self._rows(
"SELECT fold, group_concat(company_key, char(9)) AS keys FROM company_aliases "
"GROUP BY fold HAVING COUNT(*) > 1 ORDER BY fold")]
def merge_companies(self, keep: str, other: str) -> Company:
"""Fold company `other` into `keep`, in one transaction: its roles and
subscriptions move to `keep`, each agent's stance moves unless the
agent has one on `keep` already, its aliases and its name become
aliases of `keep`, and its row goes. Copies are not recomputed
(`rebuild_dupes`). Raises KeyError for an unknown company and
ValueError for merging one into itself."""
if keep == other:
raise ValueError("a company cannot be merged into itself")
with self.transaction() as conn:
rows = {r["key"]: r["name"] for r in conn.execute(
"SELECT key, name FROM companies WHERE key IN (?, ?)", (keep, other))}
for key in (keep, other):
if key not in rows:
raise KeyError(key)
conn.execute("UPDATE roles SET company_key=? WHERE company_key=?", (keep, other))
for sub in conn.execute("SELECT id, company FROM subscriptions").fetchall():
if company_key(sub["company"]) == other:
conn.execute("UPDATE subscriptions SET company=? WHERE id=?",
(rows[keep], sub["id"]))
conn.execute("UPDATE OR IGNORE agent_companies SET company_key=? WHERE company_key=?",
(keep, other))
conn.execute("DELETE FROM agent_companies WHERE company_key=?", (other,))
conn.execute("UPDATE OR IGNORE company_aliases SET company_key=? WHERE company_key=?",
(keep, other))
conn.execute("DELETE FROM company_aliases WHERE company_key=?", (other,))
self._alias(conn, keep, rows[other])
conn.execute("DELETE FROM companies WHERE key=?", (other,))
return self._company(keep)
def _company(self, key: str, agent_id: int | None = None) -> Company:
found = self.company(key, agent_id)
if found is None:
+3 -1
View File
@@ -307,10 +307,12 @@ def suggest_companies(
text = places.fold(q)
if not text:
return []
aliases = store.company_aliases()
found = [CompanySuggestion(key=key, name=names[0] if names else key, count=count,
affinity=affinity)
for key, names, count, affinity in store.open_companies(_agent_id(store, agent))
if any(_starts_a_word(text, places.fold(name)) for name in names)]
if any(_starts_a_word(text, places.fold(name)) for name in names)
or any(_starts_a_word(text, fold) for fold in aliases.get(key, ()))]
found.sort(key=lambda c: (-c.count, c.name.casefold()))
return found[:limit]
+160
View File
@@ -0,0 +1,160 @@
"""Companies as symbols: `company_aliases`, the role that lands on an alias, the
merge, and the empty enrichment columns (migration 20)."""
from __future__ import annotations
import sqlite3
from datetime import UTC, datetime
from pathlib import Path
import pytest
from app.models import company_fold
from app.storage import Store
from app.storage.schema import _add_company_aliases
from desk.models import Item
def undo_v20(conn: sqlite3.Connection) -> None:
conn.execute("DROP TABLE company_aliases")
for col in ("sector", "stage", "size_band", "founded", "last_funding",
"last_funding_currency", "facts_source", "facts_at"):
conn.execute(f"ALTER TABLE companies DROP COLUMN {col}")
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def posting(source_id: str, company: str) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title="Backend Engineer", body="",
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": company})
item.source = "ats"
return item
def company_of(store: Store, source_id: str) -> str:
row = store._row("SELECT company_key FROM roles WHERE source_id=?", (source_id,))
return str(row["company_key"])
@pytest.mark.parametrize("name, fold", [
("Ørsted", "orsted"), ("Orsted A/S", "orsted"), ("Æble ApS", "aeble"),
("Åland Oy", "aland"), ("Celonis SE", "celonis"), ("Trade Republic Bank GmbH", "trade republic bank"),
("", ""),
])
def test_the_fold_drops_marks_and_legal_forms(name: str, fold: str) -> None:
assert company_fold(name) == fold
def test_a_company_is_its_own_alias(store: Store) -> None:
store.add_company("Ørsted A/S")
assert store.company_aliases() == {"ørsted": ["orsted"]}
def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(posting("1", "Orsted"))
store.upsert_role(posting("2", "ORSTED A/S"))
store.upsert_role(posting("3", "Ørsted"))
store.upsert_role(posting("4", "Nobody"))
assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"]
assert [c.key for c in store.companies()] == ["ørsted"]
def test_an_added_alias_finds_the_company(store: Store) -> None:
store.add_company("Ørsted")
store.add_alias("ørsted", "Dong Energy")
store.upsert_role(posting("1", "Dong Energy A/S"))
assert company_of(store, "1") == "ørsted"
with pytest.raises(KeyError):
store.add_alias("nope", "X")
with pytest.raises(ValueError):
store.add_alias("ørsted", " ")
def test_renaming_and_removing_keep_the_aliases_true(store: Store) -> None:
store.add_company("Ørsted")
store.update_company("ørsted", name="Orsted Wind")
assert store.company_aliases()["ørsted"] == ["orsted", "orsted wind"]
store.remove_company("ørsted")
assert store.company_aliases() == {}
def test_duplicates_are_listed_by_fold_and_not_merged(store: Store) -> None:
store.add_company("Ørsted")
store.add_company("Orsted")
store.add_company("Other")
assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(store.companies()) == 3
# The company with the name's own key wins when a role names it.
store.upsert_role(posting("1", "Orsted"))
assert company_of(store, "1") == "orsted"
def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
store.add_company("Ørsted")
store.add_company("Orsted")
store.upsert_role(posting("1", "Orsted"))
store.upsert_role(posting("2", "Ørsted"))
sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted",
board="greenhouse")
a, b = store.save_agent("A"), store.save_agent("B")
store.set_affinity(a.id, "orsted", 2)
store.set_affinity(a.id, "ørsted", -1) # the kept company's rating wins
store.set_affinity(b.id, "orsted", 1)
kept = store.merge_companies("ørsted", "orsted")
assert kept.key == "ørsted" and store.company("orsted") is None
assert {company_of(store, s) for s in "12"} == {"ørsted"}
assert store.subscription(sub.id).company == "Ørsted"
assert store.company("ørsted", a.id).affinity == -1
assert store.company("ørsted", b.id).affinity == 1
assert store.duplicate_companies() == []
assert store.company_aliases() == {"ørsted": ["orsted"]}
# A later role from the gone spelling lands on the kept company.
store.upsert_role(posting("3", "Orsted"))
assert company_of(store, "3") == "ørsted"
with pytest.raises(ValueError):
store.merge_companies("ørsted", "ørsted")
with pytest.raises(KeyError):
store.merge_companies("ørsted", "gone")
def test_the_enrichment_columns_start_empty_and_can_be_filled(store: Store) -> None:
store.add_company("Ørsted")
row = store._row("SELECT * FROM companies WHERE key='ørsted'")
assert (row["sector"], row["stage"], row["size_band"], row["founded"],
row["last_funding"], row["last_funding_currency"], row["facts_source"],
row["facts_at"]) == ("", "", "", None, None, "", "", "")
store.update_company("ørsted", sector="Energy", founded=2006, last_funding=100,
last_funding_currency="DKK", facts_source="thehub",
facts_at="2026-09-30")
row = store._row("SELECT * FROM companies WHERE key='ørsted'")
assert (row["sector"], row["founded"], row["facts_source"]) == ("Energy", 2006, "thehub")
def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) -> None:
path = tmp_path / "old.db"
Store(path)
with sqlite3.connect(path) as conn:
undo_v20(conn)
for key, name in (("ørsted", "Ørsted"), ("orsted", "Orsted"), ("x", "X")):
conn.execute("INSERT INTO companies (key, name, created_at, updated_at) "
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
def test_the_step_runs_twice_without_harm(tmp_path: Path) -> None:
path = tmp_path / "x.db"
Store(path)
with sqlite3.connect(path) as conn:
undo_v20(conn)
_add_company_aliases(conn)
+5 -1
View File
@@ -168,13 +168,17 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
conn.execute(f"DROP TABLE {table}")
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
conn.execute("DROP TABLE company_aliases")
for column in ("sector", "stage", "size_band", "founded", "last_funding",
"last_funding_currency", "facts_source", "facts_at"):
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
conn.execute("PRAGMA user_version = 17")
migrated = Store(path)
assert migrated.migrations_applied == [18, 19]
assert migrated.migrations_applied == [18, 19, 20]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+16 -6
View File
@@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 19
assert LATEST == 20
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -88,9 +88,19 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]:
return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")]
def _undo_v20(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v19 shape: no company aliases, no
enrichment columns."""
conn.execute("DROP TABLE company_aliases")
for column in ("sector", "stage", "size_band", "founded", "last_funding",
"last_funding_currency", "facts_source", "facts_at"):
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
def _undo_v19(conn: sqlite3.Connection) -> None:
"""A fresh file put back to the v18 shape: no gazetteer, no role places,
no teaser, one level."""
_undo_v20(conn)
for table in ("role_places", "place_includes", "place_aliases", "places"):
conn.execute(f"DROP TABLE {table}")
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
@@ -176,7 +186,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -381,7 +391,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -405,7 +415,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19]
assert migrated.migrations_applied == [16, 17, 18, 19, 20]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -434,7 +444,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19]
assert migrated.migrations_applied == [17, 18, 19, 20]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -676,7 +686,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19]
assert migrated.migrations_applied == [19, 20]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
+10
View File
@@ -1670,3 +1670,13 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
"original_url": search.location, "posted_at": kept["copies"][0]["posted_at"]}]
copy = client.get(f"/api/roles/{search.uid}").json()
assert copy["dup_of"] == board.uid and copy["copies"] == []
def test_suggest_companies_matches_through_an_alias(client: TestClient, store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(posting("1", company="Orsted"))
store.add_alias("ørsted", "Dong Energy")
for q in ("orsted", "ørsted", "dong"):
res = client.get("/api/suggest/companies", params={"q": q})
assert [(c["key"], c["name"]) for c in res.json()] == [("ørsted", "Ørsted")]