diff --git a/app/models.py b/app/models.py index 562ac85..420efc2 100644 --- a/app/models.py +++ b/app/models.py @@ -644,6 +644,16 @@ STATUSES: tuple[str, ...] = get_args(Status) #: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score). AFFINITY = range(-1, 3) +#: Why a company is on the list checked by hand. `unread` is a host the app +#: could read and has no parser for yet, `robots` a site whose robots.txt +#: disallows it, `blocked` one that answers a program with a challenge, `key` +#: one whose API answers only with a key lifted from its own script, `group` a +#: company that sits inside its owner's board, `gone` one whose site is dead, +#: and `added` one put there by hand or by an import and not looked at. '' +#: is none of these: the company is read, or is not a matter for the list. +HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"] +HAND_REASONS: tuple[str, ...] = get_args(HandReason) + # -------------------------------------------------------------- posting dates @@ -924,6 +934,12 @@ class Company(Model): #: The name `ats` knows the company by, the `acme` of acme.teamtailor.com; #: "" where discovery found none. Kept for a host no parser reads yet. ats_slug: str = "" + #: Why the company is read by hand, one of `HAND_REASONS`; "" when it is + #: not on that list. + hand_reason: str = "" + #: When a person last looked at its careers page, an ISO timestamp; "" + #: for never. + checked_at: str = "" created_at: str updated_at: str diff --git a/app/storage/schema.py b/app/storage/schema.py index cb887da..c63194d 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -107,6 +107,8 @@ CREATE TABLE IF NOT EXISTS role_tags ( -- yet. country is an ISO code and city the head office's. ats_slug is the -- name that host knows the company by (the acme of acme.teamtailor.com), -- '' where its URL carries none, kept for the parser written later. +-- hand_reason is why a person reads the careers page (models.HAND_REASONS), +-- '' when nobody has to, and checked_at when one last did, '' for never. CREATE TABLE IF NOT EXISTS companies ( key TEXT PRIMARY KEY, name TEXT NOT NULL, @@ -119,7 +121,9 @@ CREATE TABLE IF NOT EXISTS companies ( ats TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', city TEXT NOT NULL DEFAULT '', - ats_slug TEXT NOT NULL DEFAULT '' + ats_slug TEXT NOT NULL DEFAULT '', + hand_reason TEXT NOT NULL DEFAULT '', + checked_at TEXT NOT NULL DEFAULT '' ); -- The targets desk's runner hands to plugins. newsdesk's shape, with its two @@ -423,6 +427,11 @@ ALTER TABLE agents DROP COLUMN intent; ALTER TABLE agents DROP COLUMN intent_key; ALTER TABLE agents DROP COLUMN intent_vec; ALTER TABLE scores ADD COLUMN prompt TEXT NOT NULL DEFAULT ''; +"""), + (15, "companies.hand_reason and checked_at: the companies read by hand are a list " + "with a reason and a date last looked at", """ +ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; +ALTER TABLE companies ADD COLUMN checked_at TEXT NOT NULL DEFAULT ''; """), ) diff --git a/app/storage/store.py b/app/storage/store.py index 1f7d22d..5a5adea 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -21,7 +21,7 @@ import numpy as np from desk.models import Item from app.models import ( - AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, + AFFINITY, HAND_REASONS, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key, @@ -94,7 +94,7 @@ _EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)" _EXCERPT_RAW = 240 _COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city", - "ats_slug") + "ats_slug", "hand_reason", "checked_at") #: A company as the store serves it: the row and the asked-for agent's affinity. _COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c @@ -1162,18 +1162,22 @@ class Store: def add_company(self, name: str, *, key: str = "", domain: str = "", status: str = "researching", careers_url: str = "", notes: str = "", ats: str = "", country: str = "", - city: str = "", ats_slug: str = "") -> Company: - """Raises ValueError on a key already taken.""" + city: str = "", ats_slug: str = "", hand_reason: str = "", + checked_at: str = "") -> Company: + """Raises ValueError on a key already taken or a `hand_reason` outside + `HAND_REASONS`.""" name = name.strip() key = key or company_key(name) if not key: raise ValueError("a company needs a name") + _check_hand_reason(hand_reason) try: with self.transaction() as conn: self._insert_company(conn, key, name, { "domain": domain, "status": status, "careers_url": careers_url, "notes": notes, "ats": ats, "country": country, "city": city, - "ats_slug": ats_slug}) + "ats_slug": ats_slug, "hand_reason": hand_reason, + "checked_at": checked_at}) except sqlite3.IntegrityError as exc: raise ValueError(f"company {key!r} already exists") from exc return self._company(key) @@ -1237,6 +1241,8 @@ class Store: unknown = set(fields) - set(_COMPANY_FIELDS) if unknown: raise ValueError(f"not a company field: {', '.join(sorted(unknown))}") + if "hand_reason" in fields: + _check_hand_reason(fields["hand_reason"]) if fields: sets = ", ".join(f"{f}=?" for f in fields) changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?", @@ -1246,7 +1252,13 @@ class Store: return self._company(key) def remove_company(self, key: str) -> bool: - return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0 + """Delete the row and every agent's stance on it, in one transaction: + `agent_companies` holds the key with no foreign key to hold it back, so + a row left behind would give a company added again under the same name + an affinity it never got. Roles and subscriptions are not touched.""" + with self.transaction() as conn: + conn.execute("DELETE FROM agent_companies WHERE company_key=?", (key,)) + return conn.execute("DELETE FROM companies WHERE key=?", (key,)).rowcount > 0 def _company(self, key: str, agent_id: int | None = None) -> Company: found = self.company(key, agent_id) @@ -1541,6 +1553,12 @@ def _check_status(status: str) -> None: f"({', '.join(STATUSES)})") +def _check_hand_reason(reason: Any) -> None: + if reason not in HAND_REASONS: + raise ValueError(f"hand_reason is one of {', '.join(repr(r) for r in HAND_REASONS)}, " + f"not {reason!r}") + + def _check_affinity(affinity: Any) -> None: if isinstance(affinity, bool) or not isinstance(affinity, int) or affinity not in AFFINITY: raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}") diff --git a/tests/test_storage.py b/tests/test_storage.py index 210fb36..0e13647 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 14 + assert LATEST == 15 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -88,10 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]: return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")] +def _undo_v15(conn: sqlite3.Connection) -> None: + """A fresh file put back to the v14 shape, without the by-hand columns of + a company, and then to the v13 one by `_undo_v14`. Every downgrade below + starts here, since v15 is the last migration.""" + conn.execute("ALTER TABLE companies DROP COLUMN hand_reason") + conn.execute("ALTER TABLE companies DROP COLUMN checked_at") + _undo_v14(conn) + + def _undo_v14(conn: sqlite3.Connection) -> None: """A fresh file put back to the v13 shape: an agent's intent and its - cached vector on the agent row, no prompts, no prompt on a score. Every - downgrade below starts here, since v14 is the last migration.""" + cached vector on the agent row, no prompts, no prompt on a score.""" conn.execute("DROP TABLE agent_prompts") conn.execute("ALTER TABLE scores DROP COLUMN prompt") conn.execute("ALTER TABLE agents ADD COLUMN intent TEXT NOT NULL DEFAULT ''") @@ -104,7 +112,7 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None: Store(fresh) Store(old) with sqlite3.connect(old) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at for column in ("permit", "languages", "extract_tries", "summary", "summarised_at", "article_text", "article_at", "read_at", "posted_at", "salary_text"): @@ -131,7 +139,7 @@ def test_a_v8_file_gains_ats_slug_and_keeps_its_companies(tmp_path: Path) -> Non path = tmp_path / "v8.db" Store(path).add_company("Acme", ats="teamtailor", country="DK") with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("ALTER TABLE companies DROP COLUMN ats_slug") conn.execute("DROP INDEX idx_roles_posted") # v10, which follows conn.execute("ALTER TABLE roles DROP COLUMN posted_at") @@ -150,7 +158,7 @@ def _downgrade_to_v4(path: Path) -> None: """A fresh file put back to the v4 shape: votes on the role, affinity on the company, no read state, no tags, no per-agent tables.""" with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("ALTER TABLE roles DROP COLUMN read_at") conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9 @@ -262,7 +270,7 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( assert before_trusted is not None and before_untrusted is not None with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("DROP INDEX idx_roles_posted") conn.execute("ALTER TABLE roles DROP COLUMN posted_at") conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12 @@ -290,7 +298,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( for item in (wrong, several, given, extracted): store.upsert_role(item) with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,)) conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", @@ -317,7 +325,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P store.replace_scores(backend.id, [(uid, 0.7, "")]) vec = np.asarray([0.6, 0.8], dtype=np.float32) with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("UPDATE agents SET intent = 'python backend', intent_key = 'k', " "intent_vec = ? WHERE id = ?", (pack(vec), backend.id)) conn.execute("UPDATE agents SET intent = ' \n ' WHERE id = ?", (blank.id,)) @@ -325,7 +333,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14] + assert migrated.migrations_applied == [14, 15] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -339,6 +347,25 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P assert not {"intent", "intent_key", "intent_vec"} & columns +def test_v15_gives_a_company_no_reason_and_no_check_and_keeps_its_row(tmp_path: Path) -> None: + fresh, path = tmp_path / "fresh.db", tmp_path / "v14.db" + Store(fresh) + Store(path).add_company("Acme", careers_url="https://acme.example/jobs", notes="met them") + with sqlite3.connect(path) as conn: + conn.execute("ALTER TABLE companies DROP COLUMN hand_reason") + conn.execute("ALTER TABLE companies DROP COLUMN checked_at") + conn.execute("PRAGMA user_version = 14") + + migrated = Store(path) + + assert migrated.migrations_applied == [15] + acme = migrated.company("acme") + assert acme is not None + assert (acme.careers_url, acme.notes, acme.hand_reason, acme.checked_at) == ( + "https://acme.example/jobs", "met them", "", "") + assert sorted(_columns(path, "companies")) == sorted(_columns(fresh, "companies")) + + def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None: ai = Prompt(name="AI leadership", text="Head of AI") fde = Prompt(name="Forward deployed", text="FDE lead") @@ -424,13 +451,13 @@ def test_posted_host_reads_board_then_provider() -> None: def test_posted_at_trusts_only_a_known_host() -> None: - assert "greenhouse" in TRUSTED_POSTED_HOSTS and "workday" not in TRUSTED_POSTED_HOSTS + assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS early = "2020-01-01T00:00:00+00:00" late = "2026-09-23T09:00:00+00:00" # A trusted host's earlier date wins over first_seen... assert posted_at(early, late, "greenhouse") == early # ...but an untrusted (or unknown) host never moves it off first_seen. - assert posted_at(early, late, "workday") == late + assert posted_at(early, late, "taleo") == late assert posted_at(early, late, "") == late @@ -982,6 +1009,37 @@ def test_companies_crud(store: Store) -> None: assert store.company("acme") is None +def test_a_company_takes_a_reason_from_the_vocabulary_and_a_check_date(store: Store) -> None: + acme = store.add_company("Acme", hand_reason="robots", checked_at="2026-09-01T00:00:00+00:00") + assert (acme.hand_reason, acme.checked_at) == ("robots", "2026-09-01T00:00:00+00:00") + assert store.add_company("Plain").hand_reason == "" + with pytest.raises(ValueError, match="hand_reason is one of"): + store.add_company("Odd", hand_reason="lazy") + assert store.company("odd") is None + assert store.update_company("acme", hand_reason="").hand_reason == "" + with pytest.raises(ValueError, match="hand_reason is one of"): + store.update_company("acme", hand_reason="lazy") + assert store.company("acme").hand_reason == "" # type: ignore[union-attr] + + +def test_removing_a_company_takes_every_agents_stance_on_it(store: Store) -> None: + store.add_company("Acme") + store.add_company("Other") + one, two = store.save_agent("One"), store.save_agent("Two") + store.set_affinity(one.id, "acme", 2) + store.set_affinity(two.id, "acme", -1) + store.set_affinity(one.id, "other", 1) + + assert store.remove_company("acme") is True + assert store.remove_company("acme") is False + + # Added again under the same name, it starts neutral for everyone. + store.add_company("Acme") + assert store.company("acme", one.id).affinity == 0 # type: ignore[union-attr] + assert store.company("acme", two.id).affinity == 0 # type: ignore[union-attr] + assert store.company("other", one.id).affinity == 1 # type: ignore[union-attr] + + def test_affinity_is_per_agent(store: Store) -> None: store.add_company("Acme", key="acme") store.add_company("Other", key="other")