From a466ea52b78a6d1cf8f3a9cae25c50993958f444 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:02:18 +0200 Subject: [PATCH 1/7] Give a company a by-hand reason and a date last checked, in migration 15 Co-Authored-By: Claude Sonnet 5.5 --- app/models.py | 16 +++++++++ app/storage/schema.py | 11 +++++- app/storage/store.py | 30 ++++++++++++---- tests/test_storage.py | 82 ++++++++++++++++++++++++++++++++++++------- 4 files changed, 120 insertions(+), 19 deletions(-) diff --git a/app/models.py b/app/models.py index 562ac85..420efc2 100644 --- a/app/models.py +++ b/app/models.py @@ -644,6 +644,16 @@ STATUSES: tuple[str, ...] = get_args(Status) #: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score). AFFINITY = range(-1, 3) +#: Why a company is on the list checked by hand. `unread` is a host the app +#: could read and has no parser for yet, `robots` a site whose robots.txt +#: disallows it, `blocked` one that answers a program with a challenge, `key` +#: one whose API answers only with a key lifted from its own script, `group` a +#: company that sits inside its owner's board, `gone` one whose site is dead, +#: and `added` one put there by hand or by an import and not looked at. '' +#: is none of these: the company is read, or is not a matter for the list. +HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"] +HAND_REASONS: tuple[str, ...] = get_args(HandReason) + # -------------------------------------------------------------- posting dates @@ -924,6 +934,12 @@ class Company(Model): #: The name `ats` knows the company by, the `acme` of acme.teamtailor.com; #: "" where discovery found none. Kept for a host no parser reads yet. ats_slug: str = "" + #: Why the company is read by hand, one of `HAND_REASONS`; "" when it is + #: not on that list. + hand_reason: str = "" + #: When a person last looked at its careers page, an ISO timestamp; "" + #: for never. + checked_at: str = "" created_at: str updated_at: str diff --git a/app/storage/schema.py b/app/storage/schema.py index cb887da..c63194d 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -107,6 +107,8 @@ CREATE TABLE IF NOT EXISTS role_tags ( -- yet. country is an ISO code and city the head office's. ats_slug is the -- name that host knows the company by (the acme of acme.teamtailor.com), -- '' where its URL carries none, kept for the parser written later. +-- hand_reason is why a person reads the careers page (models.HAND_REASONS), +-- '' when nobody has to, and checked_at when one last did, '' for never. CREATE TABLE IF NOT EXISTS companies ( key TEXT PRIMARY KEY, name TEXT NOT NULL, @@ -119,7 +121,9 @@ CREATE TABLE IF NOT EXISTS companies ( ats TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', city TEXT NOT NULL DEFAULT '', - ats_slug TEXT NOT NULL DEFAULT '' + ats_slug TEXT NOT NULL DEFAULT '', + hand_reason TEXT NOT NULL DEFAULT '', + checked_at TEXT NOT NULL DEFAULT '' ); -- The targets desk's runner hands to plugins. newsdesk's shape, with its two @@ -423,6 +427,11 @@ ALTER TABLE agents DROP COLUMN intent; ALTER TABLE agents DROP COLUMN intent_key; ALTER TABLE agents DROP COLUMN intent_vec; ALTER TABLE scores ADD COLUMN prompt TEXT NOT NULL DEFAULT ''; +"""), + (15, "companies.hand_reason and checked_at: the companies read by hand are a list " + "with a reason and a date last looked at", """ +ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; +ALTER TABLE companies ADD COLUMN checked_at TEXT NOT NULL DEFAULT ''; """), ) diff --git a/app/storage/store.py b/app/storage/store.py index 1f7d22d..5a5adea 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -21,7 +21,7 @@ import numpy as np from desk.models import Item from app.models import ( - AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, + AFFINITY, HAND_REASONS, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key, @@ -94,7 +94,7 @@ _EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)" _EXCERPT_RAW = 240 _COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city", - "ats_slug") + "ats_slug", "hand_reason", "checked_at") #: A company as the store serves it: the row and the asked-for agent's affinity. _COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c @@ -1162,18 +1162,22 @@ class Store: def add_company(self, name: str, *, key: str = "", domain: str = "", status: str = "researching", careers_url: str = "", notes: str = "", ats: str = "", country: str = "", - city: str = "", ats_slug: str = "") -> Company: - """Raises ValueError on a key already taken.""" + city: str = "", ats_slug: str = "", hand_reason: str = "", + checked_at: str = "") -> Company: + """Raises ValueError on a key already taken or a `hand_reason` outside + `HAND_REASONS`.""" name = name.strip() key = key or company_key(name) if not key: raise ValueError("a company needs a name") + _check_hand_reason(hand_reason) try: with self.transaction() as conn: self._insert_company(conn, key, name, { "domain": domain, "status": status, "careers_url": careers_url, "notes": notes, "ats": ats, "country": country, "city": city, - "ats_slug": ats_slug}) + "ats_slug": ats_slug, "hand_reason": hand_reason, + "checked_at": checked_at}) except sqlite3.IntegrityError as exc: raise ValueError(f"company {key!r} already exists") from exc return self._company(key) @@ -1237,6 +1241,8 @@ class Store: unknown = set(fields) - set(_COMPANY_FIELDS) if unknown: raise ValueError(f"not a company field: {', '.join(sorted(unknown))}") + if "hand_reason" in fields: + _check_hand_reason(fields["hand_reason"]) if fields: sets = ", ".join(f"{f}=?" for f in fields) changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?", @@ -1246,7 +1252,13 @@ class Store: return self._company(key) def remove_company(self, key: str) -> bool: - return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0 + """Delete the row and every agent's stance on it, in one transaction: + `agent_companies` holds the key with no foreign key to hold it back, so + a row left behind would give a company added again under the same name + an affinity it never got. Roles and subscriptions are not touched.""" + with self.transaction() as conn: + conn.execute("DELETE FROM agent_companies WHERE company_key=?", (key,)) + return conn.execute("DELETE FROM companies WHERE key=?", (key,)).rowcount > 0 def _company(self, key: str, agent_id: int | None = None) -> Company: found = self.company(key, agent_id) @@ -1541,6 +1553,12 @@ def _check_status(status: str) -> None: f"({', '.join(STATUSES)})") +def _check_hand_reason(reason: Any) -> None: + if reason not in HAND_REASONS: + raise ValueError(f"hand_reason is one of {', '.join(repr(r) for r in HAND_REASONS)}, " + f"not {reason!r}") + + def _check_affinity(affinity: Any) -> None: if isinstance(affinity, bool) or not isinstance(affinity, int) or affinity not in AFFINITY: raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}") diff --git a/tests/test_storage.py b/tests/test_storage.py index 210fb36..0e13647 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 14 + assert LATEST == 15 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -88,10 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]: return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")] +def _undo_v15(conn: sqlite3.Connection) -> None: + """A fresh file put back to the v14 shape, without the by-hand columns of + a company, and then to the v13 one by `_undo_v14`. Every downgrade below + starts here, since v15 is the last migration.""" + conn.execute("ALTER TABLE companies DROP COLUMN hand_reason") + conn.execute("ALTER TABLE companies DROP COLUMN checked_at") + _undo_v14(conn) + + def _undo_v14(conn: sqlite3.Connection) -> None: """A fresh file put back to the v13 shape: an agent's intent and its - cached vector on the agent row, no prompts, no prompt on a score. Every - downgrade below starts here, since v14 is the last migration.""" + cached vector on the agent row, no prompts, no prompt on a score.""" conn.execute("DROP TABLE agent_prompts") conn.execute("ALTER TABLE scores DROP COLUMN prompt") conn.execute("ALTER TABLE agents ADD COLUMN intent TEXT NOT NULL DEFAULT ''") @@ -104,7 +112,7 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None: Store(fresh) Store(old) with sqlite3.connect(old) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at for column in ("permit", "languages", "extract_tries", "summary", "summarised_at", "article_text", "article_at", "read_at", "posted_at", "salary_text"): @@ -131,7 +139,7 @@ def test_a_v8_file_gains_ats_slug_and_keeps_its_companies(tmp_path: Path) -> Non path = tmp_path / "v8.db" Store(path).add_company("Acme", ats="teamtailor", country="DK") with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("ALTER TABLE companies DROP COLUMN ats_slug") conn.execute("DROP INDEX idx_roles_posted") # v10, which follows conn.execute("ALTER TABLE roles DROP COLUMN posted_at") @@ -150,7 +158,7 @@ def _downgrade_to_v4(path: Path) -> None: """A fresh file put back to the v4 shape: votes on the role, affinity on the company, no read state, no tags, no per-agent tables.""" with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("ALTER TABLE roles DROP COLUMN read_at") conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9 @@ -262,7 +270,7 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( assert before_trusted is not None and before_untrusted is not None with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("DROP INDEX idx_roles_posted") conn.execute("ALTER TABLE roles DROP COLUMN posted_at") conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12 @@ -290,7 +298,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( for item in (wrong, several, given, extracted): store.upsert_role(item) with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,)) conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", @@ -317,7 +325,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P store.replace_scores(backend.id, [(uid, 0.7, "")]) vec = np.asarray([0.6, 0.8], dtype=np.float32) with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("UPDATE agents SET intent = 'python backend', intent_key = 'k', " "intent_vec = ? WHERE id = ?", (pack(vec), backend.id)) conn.execute("UPDATE agents SET intent = ' \n ' WHERE id = ?", (blank.id,)) @@ -325,7 +333,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14] + assert migrated.migrations_applied == [14, 15] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -339,6 +347,25 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P assert not {"intent", "intent_key", "intent_vec"} & columns +def test_v15_gives_a_company_no_reason_and_no_check_and_keeps_its_row(tmp_path: Path) -> None: + fresh, path = tmp_path / "fresh.db", tmp_path / "v14.db" + Store(fresh) + Store(path).add_company("Acme", careers_url="https://acme.example/jobs", notes="met them") + with sqlite3.connect(path) as conn: + conn.execute("ALTER TABLE companies DROP COLUMN hand_reason") + conn.execute("ALTER TABLE companies DROP COLUMN checked_at") + conn.execute("PRAGMA user_version = 14") + + migrated = Store(path) + + assert migrated.migrations_applied == [15] + acme = migrated.company("acme") + assert acme is not None + assert (acme.careers_url, acme.notes, acme.hand_reason, acme.checked_at) == ( + "https://acme.example/jobs", "met them", "", "") + assert sorted(_columns(path, "companies")) == sorted(_columns(fresh, "companies")) + + def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None: ai = Prompt(name="AI leadership", text="Head of AI") fde = Prompt(name="Forward deployed", text="FDE lead") @@ -424,13 +451,13 @@ def test_posted_host_reads_board_then_provider() -> None: def test_posted_at_trusts_only_a_known_host() -> None: - assert "greenhouse" in TRUSTED_POSTED_HOSTS and "workday" not in TRUSTED_POSTED_HOSTS + assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS early = "2020-01-01T00:00:00+00:00" late = "2026-09-23T09:00:00+00:00" # A trusted host's earlier date wins over first_seen... assert posted_at(early, late, "greenhouse") == early # ...but an untrusted (or unknown) host never moves it off first_seen. - assert posted_at(early, late, "workday") == late + assert posted_at(early, late, "taleo") == late assert posted_at(early, late, "") == late @@ -982,6 +1009,37 @@ def test_companies_crud(store: Store) -> None: assert store.company("acme") is None +def test_a_company_takes_a_reason_from_the_vocabulary_and_a_check_date(store: Store) -> None: + acme = store.add_company("Acme", hand_reason="robots", checked_at="2026-09-01T00:00:00+00:00") + assert (acme.hand_reason, acme.checked_at) == ("robots", "2026-09-01T00:00:00+00:00") + assert store.add_company("Plain").hand_reason == "" + with pytest.raises(ValueError, match="hand_reason is one of"): + store.add_company("Odd", hand_reason="lazy") + assert store.company("odd") is None + assert store.update_company("acme", hand_reason="").hand_reason == "" + with pytest.raises(ValueError, match="hand_reason is one of"): + store.update_company("acme", hand_reason="lazy") + assert store.company("acme").hand_reason == "" # type: ignore[union-attr] + + +def test_removing_a_company_takes_every_agents_stance_on_it(store: Store) -> None: + store.add_company("Acme") + store.add_company("Other") + one, two = store.save_agent("One"), store.save_agent("Two") + store.set_affinity(one.id, "acme", 2) + store.set_affinity(two.id, "acme", -1) + store.set_affinity(one.id, "other", 1) + + assert store.remove_company("acme") is True + assert store.remove_company("acme") is False + + # Added again under the same name, it starts neutral for everyone. + store.add_company("Acme") + assert store.company("acme", one.id).affinity == 0 # type: ignore[union-attr] + assert store.company("acme", two.id).affinity == 0 # type: ignore[union-attr] + assert store.company("other", one.id).affinity == 1 # type: ignore[union-attr] + + def test_affinity_is_per_agent(store: Store) -> None: store.add_company("Acme", key="acme") store.add_company("Other", key="other") From b3cb93d5aa6b9cfbfb88c1ffc96283ef87dad43d Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:02:19 +0200 Subject: [PATCH 2/7] Keep a typed careers link, seed the by-hand reason and add the by-hand commands to app companies Co-Authored-By: Claude Sonnet 5.5 --- app/cli.py | 127 +++++++++++++++++++++-- app/jobs/sources.py | 125 +++++++++++++++++++++- tests/test_sources.py | 235 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 472 insertions(+), 15 deletions(-) diff --git a/app/cli.py b/app/cli.py index 6017af6..35b3cbe 100644 --- a/app/cli.py +++ b/app/cli.py @@ -3,7 +3,7 @@ app serve [--host H] [--port P] [--reload] app job pull|extract|embed|score [--payload JSON] app worker [--workers http=8,cpu=1] [--until-idle] - app companies seed|add|set-board|detach|discover|check + app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check app searches add|list|remove app agents list|prompt app stages [extract|summarise on|off] @@ -44,6 +44,8 @@ if str(_ROOT) not in sys.path: def build_parser() -> argparse.ArgumentParser: + from app.models import HAND_REASONS + parser = argparse.ArgumentParser( prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.") parser.add_argument("-c", "--config", default=None, @@ -71,10 +73,34 @@ def build_parser() -> argparse.ArgumentParser: seed = companies_sub.add_parser( "seed", help="import seed/companies.json, and the boards it names for rows without one") seed.add_argument("--path", default="seed/companies.json") + # One word after `add` is a company for the list checked by hand, two are + # a board and its company, which is what `add` meant before that list. add = companies_sub.add_parser( - "add", help="follow a company by its board, attaching or replacing an existing one's") - add.add_argument("board", help="host:slug, e.g. greenhouse:acme") - add.add_argument("name") + "add", help="add NAME to the list checked by hand, or with BOARD NAME follow a " + "company by its board, attaching or replacing an existing one's") + add.add_argument("first", metavar="NAME|BOARD", + help="the company, or host:slug, e.g. greenhouse:acme, before its name") + add.add_argument("second", metavar="NAME", nargs="?") + add.add_argument("--careers-url", default="", help="its careers page, http or https") + add.add_argument("--note", default="") + add.add_argument("--reason", default="added", choices=HAND_REASONS, metavar="REASON", + help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}") + edit = companies_sub.add_parser("edit", help="change a company's name, link, note or reason") + edit.add_argument("key") + edit.add_argument("--name") + edit.add_argument("--careers-url") + edit.add_argument("--note") + edit.add_argument("--reason", choices=HAND_REASONS, metavar="REASON", + help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}") + checked = companies_sub.add_parser( + "checked", help="record that its careers page was looked at just now") + checked.add_argument("key") + remove = companies_sub.add_parser( + "remove", help="delete a company with no board on and no open roles") + remove.add_argument("key") + companies_sub.add_parser( + "by-hand", help="the companies checked by hand, never checked first, then the " + "longest ago") set_board = companies_sub.add_parser( "set-board", help="attach a board to a company, or replace the one it has") set_board.add_argument("key", help="the company key, e.g. trade-republic") @@ -274,7 +300,15 @@ def run_companies(args: argparse.Namespace) -> int: if args.action == "seed": return seed_companies(store, args.path) if args.action == "add": - return add_company(store, args.board, args.name) + return add_company(store, args) + if args.action == "edit": + return edit_company(store, args) + if args.action == "checked": + return company_checked(store, args.key) + if args.action == "remove": + return remove_company(store, args.key) + if args.action == "by-hand": + return list_by_hand(store) if args.action == "set-board": return set_board(store, args.key, args.board) if args.action == "detach": @@ -318,7 +352,25 @@ def discover_company(store: Store, name: str, domain: str, *, add: bool) -> int: return 0 -def add_company(store: Store, board: str, name: str) -> int: +def add_company(store: Store, args: argparse.Namespace) -> int: + """`app companies add "Acme" [--careers-url URL] [--note TEXT] [--reason + REASON]` puts a company with no board on the list checked by hand; + `app companies add greenhouse:acme "Acme"` follows it by its board.""" + if args.second is not None: + return add_company_by_board(store, args.first, args.second) + from app import jobs + + try: + company = jobs.sources.add_by_hand(store, args.first, careers_url=args.careers_url, + notes=args.note, hand_reason=args.reason) + except ValueError as exc: + print(f"companies add: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": company.key, "hand_reason": company.hand_reason})) + return 0 + + +def add_company_by_board(store: Store, board: str, name: str) -> int: """`app companies add greenhouse:acme "Acme"`: fetch the board first and refuse if it does not answer or is not a known host. A company that exists gets the board attached or replaced. Port of seedgen's `-add`.""" @@ -334,6 +386,69 @@ def add_company(store: Store, board: str, name: str) -> int: return 0 +def edit_company(store: Store, args: argparse.Namespace) -> int: + """`app companies edit acme --careers-url https://acme.example/jobs`: only + the options given change; see `jobs.sources.edit_company`.""" + from app import jobs + + given = {"name": args.name, "careers_url": args.careers_url, "notes": args.note, + "hand_reason": args.reason} + try: + company = jobs.sources.edit_company( + store, args.key, **{f: v for f, v in given.items() if v is not None}) + except KeyError: + print(f"companies edit: no company {args.key!r}", file=sys.stderr) + return 1 + except ValueError as exc: + print(f"companies edit: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": company.key, "name": company.name, + "careers_url": company.careers_url, "notes": company.notes, + "hand_reason": company.hand_reason})) + return 0 + + +def company_checked(store: Store, key: str) -> int: + """`app companies checked acme`: its careers page was looked at just now.""" + if store.company(key) is None: + print(f"companies checked: no company {key!r}", file=sys.stderr) + return 1 + company = store.update_company(key, checked_at=store.now()) + print(json.dumps({"key": key, "checked_at": company.checked_at})) + return 0 + + +def remove_company(store: Store, key: str) -> int: + """`app companies remove acme`: refused for a company still read, see + `jobs.sources.remove_company`.""" + from app import jobs + + try: + jobs.sources.remove_company(store, key) + except KeyError: + print(f"companies remove: no company {key!r}", file=sys.stderr) + return 1 + except ValueError as exc: + print(f"companies remove: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": key, "removed": True})) + return 0 + + +def list_by_hand(store: Store) -> int: + """`app companies by-hand`: the companies with no board shown that are not + known from a search only, one line each, never checked first and then the + longest ago, as the Sources page lists them; see `jobs.sources.by_hand`.""" + from app import jobs + + rows = jobs.sources.by_hand(store) + for c in rows: + checked = c.checked_at[:10] or "never" + print(f"{c.name:30} {c.hand_reason or '-':8} {checked:10} {c.careers_url}") + print(f"{len(rows)} by hand") + return 0 + + def set_board(store: Store, key: str, board: str) -> int: """`app companies set-board trade-republic greenhouse:traderepublic`: attach the board to an existing company or replace the one it has.""" diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 4eedbea..9e566b6 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -15,9 +15,10 @@ import json import os from pathlib import Path from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit from app.config import Config -from app.models import Subscription, company_key +from app.models import HAND_REASONS, Company, Subscription, company_key from app.storage import Store from app.jobs.pull import FAILING, search_target @@ -150,12 +151,116 @@ def add_discovered(store: Store, name: str, domain: str, if sub is not None and sub.show: raise ValueError(f"{name} is read from {sub.target} and discovery found no " "readable board; nothing changed") + # A link somebody typed is kept: discovery's is only a guess at the page. store.update_company(key, ats=found.ats, ats_slug=found.ats_slug, - careers_url=found.careers_url or company.careers_url, + careers_url=company.careers_url or found.careers_url, **({"domain": domain} if not company.domain else {})) return key, 0 +def known_from_search_only(company: Company, ever_boarded: set[str]) -> bool: + """A row that exists only because a search listed its roles and somebody + judged it (`Store.set_affinity`): no board, not even a detached one + (`ever_boarded`, the keys of every `ats` subscription), and none of the + facts every way of following a company or listing it by hand writes.""" + return not (company.key in ever_boarded or company.ats or company.careers_url + or company.domain or company.hand_reason) + + +def by_hand(store: Store) -> list[Company]: + """The list checked by hand: companies with no board shown that are not + known from a search only. Never checked first, then the longest ago; the + name breaks a tie.""" + boards = store.company_boards() + ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} + rows = [c for c in store.companies() + if c.key not in boards and not known_from_search_only(c, ever_boarded)] + return sorted(rows, key=lambda c: (c.checked_at, c.name.casefold())) + + +class Conflict(ValueError): + """A change refused because of what the company already is, not because of + what was asked for: the pages answer it with 409, not 422.""" + + +def check_careers_url(url: str) -> str: + """`url` stripped; ValueError unless it is empty or an http(s) link. The + page puts it in an `href`, where `javascript:` would run.""" + url = url.strip() + if url: + parts = urlsplit(url) + if parts.scheme not in ("http", "https") or not parts.netloc: + raise ValueError("The careers link must start with http:// or https://.") + return url + + +def add_by_hand(store: Store, name: str, *, careers_url: str = "", notes: str = "", + hand_reason: str = "added") -> Company: + """A company with no board, put on the list checked by hand. Raises + Conflict when the key its name folds to is taken, ValueError for a name + with no letters, a link that is no http(s) URL or a reason outside + `HAND_REASONS`.""" + link = check_careers_url(careers_url) + key = company_key(name) + if key and store.company(key) is not None: + raise Conflict(f"{name.strip()} is on the list already as {key}.") + return store.add_company(name, key=key, careers_url=link, notes=notes, + hand_reason=hand_reason) + + +def edit_company(store: Store, key: str, **fields: str) -> Company: + """Change a company's `name`, `careers_url`, `notes` or `hand_reason`; the + key never changes. A new name is Conflict for a company with a board + subscription, shown or not, since the subscription is found by the key + its company name folds to and would be lost. Raises KeyError for an + unknown company and ValueError for a blank name, a link that is no http(s) + URL or a reason outside `HAND_REASONS`.""" + unknown = set(fields) - {"name", "careers_url", "notes", "hand_reason"} + if unknown: + raise ValueError(f"not a field to edit: {', '.join(sorted(unknown))}") + company = store.company(key) + if company is None: + raise KeyError(key) + if "careers_url" in fields: + fields["careers_url"] = check_careers_url(fields["careers_url"]) + if "name" in fields: + fields["name"] = fields["name"].strip() + if not fields["name"]: + raise ValueError("A company needs a name.") + if fields["name"] != company.name and store.company_subscription(key) is not None: + raise Conflict(f"{company.name} has a board, which is found by its name; " + "detach the board first.") + return store.update_company(key, **fields) + + +def remove_company(store: Store, key: str) -> None: + """Delete company `key` and every agent's stance on it. Raises KeyError + for an unknown company and Conflict for one still read: a company with + a board on (`show`, as `detach_board` decides) or with open roles, which + would otherwise vanish from the list with nothing said. A detached board + is no board, since nothing reads it and its roles are closed; its + subscription row stays, and follows the company if it comes back.""" + if store.company(key) is None: + raise KeyError(key) + sub = store.company_subscription(key) + if sub is not None and sub.show: + raise Conflict(f"{key} is read from {sub.target}; detach the board first") + if store.open_role_counts().get(key): + raise Conflict(f"{key} has open roles; detach its board first") + store.remove_company(key) + + +def _fill_empty(store: Store, key: str, **facts: str) -> None: + """Write each of `facts` to company `key` where the stored value is + empty: what a seed may add to a row that exists, since a person's link or + reason is never overwritten.""" + company = store.company(key) + assert company is not None + fill = {f: v for f, v in facts.items() if v and not getattr(company, f)} + if fill: + store.update_company(key, **fill) + + def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: """Import `seed/companies.json`: one `companies` row per entry, and its `board` through `set_board` where the entry names one. An entry without @@ -167,8 +272,10 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: existing row that never had a board gets the one the seed now names, so a seed update that gains boards reaches a database made before it. A row's board, once it has one (detached counts), and its affinity, notes and - status are never touched. The seed's boards are not fetched again: they - were read once when the seed was written, and `app companies check` + status are never touched; its `careers_url` and `hand_reason` are filled + where empty, so the seed can hand a row its link and its reason but never + overwrite the one a person set. The seed's boards are not fetched again: + they were read once when the seed was written, and `app companies check` finds the dead ones. Returns counts, and `problems`, one line per entry that could not land whole.""" from plugins.ats import is_known_target @@ -186,8 +293,15 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: board = "" facts = {f: str(row.get(f) or "") for f in ("domain", "country", "city", "careers_url", "ats", "ats_slug")} + reason = str(row.get("hand_reason") or "") + if reason not in HAND_REASONS: + problems.append(f"{key}: {reason!r} is not a hand_reason; kept without it") + reason = "" exists = store.company(key) is not None + if exists: + _fill_empty(store, key, hand_reason=reason) if exists and (not board or store.company_subscription(key) is not None): + _fill_empty(store, key, careers_url=facts["careers_url"]) skipped += 1 continue if board: @@ -200,13 +314,14 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: except ValueError as exc: problems.append(f"{key}: {exc}") if exists: + _fill_empty(store, key, careers_url=facts["careers_url"]) skipped += 1 continue # No board, or one that clashed: the company is still a row, and a # clashing board's host and slug are what discovery found. if board: facts["ats"], _, facts["ats_slug"] = board.partition(":") - store.add_company(name, key=key, **facts) + store.add_company(name, key=key, hand_reason=reason, **facts) added += 1 return {"added": added, "boards": boards, "skipped": skipped, "total": len(entries), "problems": problems} diff --git a/tests/test_sources.py b/tests/test_sources.py index 7665d3f..dfedad0 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -110,7 +110,7 @@ def test_a_board_that_does_not_answer_or_is_unknown_is_refused( with pytest.raises(ValueError, match="did not answer"): sources.set_board(store, "", "greenhouse:gone", name="Gone") with pytest.raises(ValueError, match="host:slug"): - sources.set_board(store, "", "workday:gone", name="Gone") + sources.set_board(store, "", "taleo:gone", name="Gone") with pytest.raises(ValueError, match="needs a name"): sources.set_board(store, "nobody", "greenhouse:acme") with pytest.raises(ValueError, match="is not the company"): @@ -161,20 +161,34 @@ def test_add_discovered_records_an_unreadable_host_with_its_slug( careers_url="https://acme.example/careers") assert sources.add_discovered(store, "Acme", "acme.example", found) == ("acme", 0) store.update_company("acme", notes="keep") - newer = Discovery(ats="workday", ats_slug="acme-inc", + newer = Discovery(ats="taleo", ats_slug="acme-inc", careers_url="https://acme.example/jobs") assert sources.add_discovered(store, "Acme", "acme.example", newer) == ("acme", 0) acme = store.company("acme") assert acme is not None + # The host and slug follow what discovery found last; the link, once + # there, is kept, since a person may have typed it. assert (acme.ats, acme.ats_slug, acme.careers_url, acme.notes) == ( - "workday", "acme-inc", "https://acme.example/jobs", "keep") + "taleo", "acme-inc", "https://acme.example/careers", "keep") + + +def test_add_discovered_fills_a_link_only_where_there_is_none( + store: Store, boards: Boards) -> None: + store.add_company("Acme", ats="teamtailor") + found = Discovery(ats="taleo", ats_slug="acme", careers_url="https://acme.example/jobs") + sources.add_discovered(store, "Acme", "acme.example", found) + assert store.company("acme").careers_url == "https://acme.example/jobs" # type: ignore[union-attr] + + store.update_company("acme", careers_url="https://typed.example/by-hand") + sources.add_discovered(store, "Acme", "acme.example", found) + assert store.company("acme").careers_url == "https://typed.example/by-hand" # type: ignore[union-attr] def test_add_discovered_keeps_a_board_when_nothing_readable_was_found( store: Store, boards: Boards) -> None: sources.set_board(store, "", "greenhouse:acme", name="Acme") with pytest.raises(ValueError, match="nothing changed"): - sources.add_discovered(store, "Acme", "acme.example", Discovery(ats="workday")) + sources.add_discovered(store, "Acme", "acme.example", Discovery(ats="taleo")) acme = store.company("acme") assert acme is not None and acme.ats == "greenhouse" @@ -240,6 +254,143 @@ def test_a_seed_board_that_clashes_still_lands_the_company( assert store.company_subscription("acme-two") is None +def test_a_seed_rerun_fills_an_empty_link_and_reason_and_never_overwrites_one( + store: Store, boards: Boards, tmp_path: Path) -> None: + store.add_company("Bare", key="bare") + store.add_company("Typed", key="typed", careers_url="https://typed.example/by-hand", + hand_reason="blocked", notes="mine") + seed = _seed(tmp_path, + {"name": "Bare", "careers_url": "https://bare.example/jobs", + "hand_reason": "unread"}, + {"name": "Typed", "careers_url": "https://seed.example/jobs", + "hand_reason": "robots"}, + {"name": "Fresh", "careers_url": "https://fresh.example/jobs", + "hand_reason": "key"}) + + result = sources.seed_companies(store, seed) + + assert (result["added"], result["skipped"], result["problems"]) == (1, 2, []) + got = {k: (c.careers_url, c.hand_reason, c.notes) for k in ("bare", "typed", "fresh") + if (c := store.company(k)) is not None} + assert got == { + "bare": ("https://bare.example/jobs", "unread", ""), + "typed": ("https://typed.example/by-hand", "blocked", "mine"), + "fresh": ("https://fresh.example/jobs", "key", "")} + # Run again, nothing moves. + assert sources.seed_companies(store, seed)["added"] == 0 + assert store.company("bare").hand_reason == "unread" # type: ignore[union-attr] + + +def test_a_seed_reason_outside_the_vocabulary_is_a_problem_not_a_row_value( + store: Store, boards: Boards, tmp_path: Path) -> None: + result = sources.seed_companies( + store, _seed(tmp_path, {"name": "Odd", "hand_reason": "lazy"})) + assert result["problems"] == ["odd: 'lazy' is not a hand_reason; kept without it"] + assert store.company("odd").hand_reason == "" # type: ignore[union-attr] + + +def test_a_seed_link_reaches_a_row_whose_board_clashes( + store: Store, boards: Boards, tmp_path: Path) -> None: + sources.set_board(store, "acme", "greenhouse:acme", name="Acme") + store.add_company("Twin", key="twin") + result = sources.seed_companies(store, _seed( + tmp_path, {"name": "Twin", "board": "greenhouse:acme", + "careers_url": "https://twin.example/jobs"})) + assert len(result["problems"]) == 1 + assert store.company("twin").careers_url == "https://twin.example/jobs" # type: ignore[union-attr] + + +# ------------------------------------------------------------- by hand + +def test_a_company_added_by_hand_has_no_board_and_a_checked_link(store: Store) -> None: + company = sources.add_by_hand(store, "Revolut Ltd", careers_url=" https://r.example/jobs ", + notes="login wall") + assert (company.key, company.hand_reason, company.careers_url, company.notes) == ( + "revolut", "added", "https://r.example/jobs", "login wall") + assert store.company_subscription("revolut") is None + with pytest.raises(sources.Conflict, match="on the list already"): + sources.add_by_hand(store, "REVOLUT") + for link in ("javascript:alert(1)", "r.example/jobs", "https://", "ftp://r.example"): + with pytest.raises(ValueError, match="http"): + sources.add_by_hand(store, "Other", careers_url=link) + with pytest.raises(ValueError, match="hand_reason"): + sources.add_by_hand(store, "Other", hand_reason="lazy") + with pytest.raises(ValueError, match="needs a name"): + sources.add_by_hand(store, " ") + assert [c.key for c in store.companies()] == ["revolut"] + + +def test_edit_company_changes_what_it_is_given_and_renames_only_without_a_board( + store: Store, boards: Boards) -> None: + sources.add_by_hand(store, "Acme", notes="old") + edited = sources.edit_company(store, "acme", name="Acme GmbH", hand_reason="gone") + assert (edited.key, edited.name, edited.notes, edited.hand_reason) == ( + "acme", "Acme GmbH", "old", "gone") + with pytest.raises(ValueError, match="needs a name"): + sources.edit_company(store, "acme", name=" ") + with pytest.raises(ValueError, match="not a field"): + sources.edit_company(store, "acme", ats="lever") + with pytest.raises(KeyError): + sources.edit_company(store, "nobody", notes="x") + + sources.set_board(store, "acme", "greenhouse:acme") + with pytest.raises(sources.Conflict, match="detach"): + sources.edit_company(store, "acme", name="Acme Labs") + # The name it has, and any other field, are no rename. + assert sources.edit_company(store, "acme", name="Acme GmbH", notes="new").notes == "new" + sources.detach_board(store, "acme") + with pytest.raises(sources.Conflict): + sources.edit_company(store, "acme", name="Acme Labs") + + +def test_remove_company_refuses_a_board_shown_and_open_roles( + store: Store, boards: Boards) -> None: + agent = store.save_agent("A") + sources.add_by_hand(store, "Hand") + store.set_affinity(agent.id, "hand", 2) + sources.remove_company(store, "hand") + assert store.company("hand") is None + assert store.company("hand", agent.id) is None + with pytest.raises(KeyError): + sources.remove_company(store, "hand") + + sub, _found, _closed = sources.set_board(store, "acme", "greenhouse:acme", name="Acme") + with pytest.raises(sources.Conflict, match="detach the board"): + sources.remove_company(store, "acme") + store.upsert_role(posting("1"), sub) + sources.detach_board(store, "acme") # closes the role: no board, none open + sources.remove_company(store, "acme") + assert store.company("acme") is None + # The detached subscription stays and is found again by a company of that name. + assert store.company_subscription("acme") is not None + + store.add_company("Open", key="open") + other = store.add_subscription("search", "jobindex:x", label="x", board="jobindex") + item = posting("2") + item.raw = {"company": "Open"} + store.upsert_role(item, other) + with pytest.raises(ValueError, match="open roles"): + sources.remove_company(store, "open") + + +def test_by_hand_lists_the_unboarded_never_checked_first( + store: Store, boards: Boards) -> None: + sources.add_by_hand(store, "Zeta") + sources.add_by_hand(store, "Beta") + sources.add_by_hand(store, "Old") + sources.add_by_hand(store, "Recent") + store.update_company("old", checked_at="2026-08-01T00:00:00+00:00") + store.update_company("recent", checked_at="2026-09-20T00:00:00+00:00") + sources.set_board(store, "read", "greenhouse:acme", name="Read") # a board: not listed + store.add_company("Pleo", key="pleo", ats="teamtailor") # unread, no reason: listed + store.add_company("Quiet", key="quiet") # from a search only + sources.set_board(store, "gone", "ashby:acme", name="Gone") + sources.detach_board(store, "gone") # detached: listed + + assert [c.key for c in sources.by_hand(store)] == [ + "beta", "gone", "pleo", "zeta", "old", "recent"] + + # -------------------------------------------------------------------- CLI def _cli(tmp_path: Path, *argv: str) -> int: @@ -289,6 +440,82 @@ def test_cli_seed_prints_counts_and_problems( assert "companies seed: odd: 'nosuch:odd' is not a known board" in captured.err +def test_cli_add_with_one_name_lists_a_company_by_hand_and_with_two_follows_a_board( + store: Store, boards: Boards, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + assert _cli(tmp_path, "add", "Revolut", "--careers-url", "https://r.example/jobs", + "--note", "login wall") == 0 + assert json.loads(capsys.readouterr().out) == {"key": "revolut", "hand_reason": "added"} + revolut = store.company("revolut") + assert revolut is not None + assert (revolut.careers_url, revolut.notes, revolut.hand_reason) == ( + "https://r.example/jobs", "login wall", "added") + assert _cli(tmp_path, "add", "Meta", "--reason", "blocked") == 0 + assert store.company("meta").hand_reason == "blocked" # type: ignore[union-attr] + capsys.readouterr() + + assert _cli(tmp_path, "add", "Revolut") == 1 + assert "on the list already" in capsys.readouterr().err + assert _cli(tmp_path, "add", "Odd", "--careers-url", "javascript:alert(1)") == 1 + assert "http" in capsys.readouterr().err + with pytest.raises(SystemExit): # argparse refuses the reason + _cli(tmp_path, "add", "Odd", "--reason", "lazy") + capsys.readouterr() + assert store.company("odd") is None + + # Two words are the board and the company, as before. + assert _cli(tmp_path, "add", "greenhouse:acme", "Acme") == 0 + assert json.loads(capsys.readouterr().out) == { + "key": "acme", "board": "greenhouse:acme", "roles": 3} + + +def test_cli_edit_checked_and_remove_a_company_by_hand( + store: Store, boards: Boards, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + store.add_company("Acme", key="acme", notes="keep", hand_reason="unread") + assert _cli(tmp_path, "edit", "acme", "--careers-url", "https://acme.example/jobs", + "--reason", "robots") == 0 + assert json.loads(capsys.readouterr().out) == { + "key": "acme", "name": "Acme", "careers_url": "https://acme.example/jobs", + "notes": "keep", "hand_reason": "robots"} + assert _cli(tmp_path, "edit", "acme", "--note", "") == 0 + assert store.company("acme").notes == "" # type: ignore[union-attr] + assert _cli(tmp_path, "edit", "acme", "--careers-url", "ftp://x") == 1 + assert "http" in capsys.readouterr().err + assert _cli(tmp_path, "edit", "nobody", "--note", "x") == 1 + assert "no company 'nobody'" in capsys.readouterr().err + + assert _cli(tmp_path, "checked", "acme") == 0 + stamp = json.loads(capsys.readouterr().out)["checked_at"] + assert stamp and store.company("acme").checked_at == stamp # type: ignore[union-attr] + assert _cli(tmp_path, "checked", "nobody") == 1 + capsys.readouterr() + + sources.set_board(store, "read", "greenhouse:acme", name="Read") + assert _cli(tmp_path, "remove", "read") == 1 + assert "detach the board first" in capsys.readouterr().err + assert _cli(tmp_path, "remove", "acme") == 0 + assert store.company("acme") is None + assert _cli(tmp_path, "remove", "acme") == 1 + + +def test_cli_by_hand_prints_never_checked_first_then_the_oldest_check( + store: Store, boards: Boards, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + store.add_company("Recent", careers_url="https://recent.example/jobs", + hand_reason="gone", checked_at="2026-09-20T08:00:00+00:00") + store.add_company("Old", hand_reason="robots", checked_at="2026-08-01T08:00:00+00:00") + store.add_company("Never", careers_url="https://never.example/jobs", hand_reason="added") + sources.set_board(store, "read", "greenhouse:acme", name="Read") + + assert _cli(tmp_path, "by-hand") == 0 + lines = capsys.readouterr().out.splitlines() + assert [ln.split()[0] for ln in lines[:-1]] == ["Never", "Old", "Recent"] + assert lines[0].split() == ["Never", "added", "never", "https://never.example/jobs"] + assert lines[1].split() == ["Old", "robots", "2026-08-01"] + assert lines[-1] == "3 by hand" + + def test_replacing_or_detaching_a_board_forgets_that_it_was_failing( store: Store, boards: Boards) -> None: from app.jobs.pull import FAILING, failing_boards From 28afc334c341700905441783092c4fab599ac673 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:02:19 +0200 Subject: [PATCH 3/7] Add, edit, check and delete a company by hand through the API, and show the by-hand fields in the list Co-Authored-By: Claude Sonnet 5.5 --- app/web/api.py | 122 ++++++++++++++++++++++++++++++----- tests/test_web.py | 161 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 259 insertions(+), 24 deletions(-) diff --git a/app/web/api.py b/app/web/api.py index 229f8b6..e27a3e2 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -10,6 +10,7 @@ This is roledesk's own API, over the same store. Every refusal is from __future__ import annotations from typing import Annotated, Any, Literal +from urllib.parse import urlsplit from fastapi import APIRouter, HTTPException, Query, Response from pydantic import BaseModel, Field @@ -17,7 +18,7 @@ from pydantic import BaseModel, Field from app import jobs from app.models import ( AFFINITY, Agent, Application, Company, Gate, Limits, Prompts, Role, RoleRow, - ScoredRole, Status, Subscription, Weights, company_key, + HandReason, ScoredRole, Status, Subscription, Weights, company_key, ) from app.storage import Store from app.web.deps import ConfigDep, StoreDep @@ -303,11 +304,17 @@ class CompanyOut(BaseModel): the board has failed on every fetch and why it last did, both empty while it answers. `from_search` marks a row that exists only because a search listed its roles and somebody judged it (`Store.set_affinity`): - no board, not even a detached one, and no `ats`, `careers_url` or - `domain`, which every way of following a company writes one of.""" + no board, not even a detached one, and no `ats`, `careers_url`, `domain` + or `hand_reason`, which every way of following a company or listing it by + hand writes one of (`jobs.sources.known_from_search_only`). `by_hand` + marks a company on the list checked by hand: no board shown and not + `from_search`. `hand_reason` says why (`app.models.HAND_REASONS`) and + `checked_at` when a person last looked at its careers page, "" for never.""" key: str name: str + domain: str = "" + notes: str = "" board: str subscription_id: int | None = None on: bool | None = None @@ -320,6 +327,9 @@ class CompanyOut(BaseModel): failing_since: str = "" failing_reason: str = "" from_search: bool = False + hand_reason: str = "" + checked_at: str = "" + by_hand: bool = False def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: @@ -337,29 +347,109 @@ def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: return boards -@router.get("/companies") -def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut]: - """Every company, with `affinity` as `agent` (the default agent when - absent) has it; 0 throughout when there is no agent.""" +def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]: boards = _company_boards(store) ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} counts = store.open_role_counts() failing = jobs.pull.failing_boards(store) out = [] - for c in store.companies(_agent_id(store, agent)): + for c in store.companies(agent): sub_id, host, on, target = boards.get(c.key, (None, "", None, "")) since, reason = failing.get(target, ("", "")) - out.append(CompanyOut(key=c.key, name=c.name, board=host, - subscription_id=sub_id, on=on, affinity=c.affinity, - open_roles=counts.get(c.key, 0), ats=c.ats, - ats_slug=c.ats_slug, careers_url=c.careers_url, - country=c.country, - failing_since=since, failing_reason=reason, - from_search=not (c.key in ever_boarded or c.ats - or c.careers_url or c.domain))) + from_search = jobs.sources.known_from_search_only(c, ever_boarded) + out.append(CompanyOut(key=c.key, name=c.name, domain=c.domain, notes=c.notes, + board=host, subscription_id=sub_id, on=on, + affinity=c.affinity, open_roles=counts.get(c.key, 0), + ats=c.ats, ats_slug=c.ats_slug, careers_url=c.careers_url, + country=c.country, failing_since=since, failing_reason=reason, + from_search=from_search, hand_reason=c.hand_reason, + checked_at=c.checked_at, + by_hand=c.key not in boards and not from_search)) return out +def _one_company(store: Store, key: str) -> CompanyOut: + """`key` as the list shows it, with the default agent's affinity.""" + return next(c for c in _company_outs(store, _agent_id(store, None)) if c.key == key) + + +@router.get("/companies") +def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut]: + """Every company, with `affinity` as `agent` (the default agent when + absent) has it; 0 throughout when there is no agent.""" + return _company_outs(store, _agent_id(store, agent)) + + +class NewCompanyBody(BaseModel): + name: str + careers_url: str = "" + notes: str = "" + hand_reason: HandReason = "added" + + +@router.post("/companies", status_code=201) +def add_company(body: NewCompanyBody, store: StoreDep) -> CompanyOut: + """A company put on the list checked by hand: no board, `hand_reason` + `added` unless it says another. 409 when its key, from the name, is taken.""" + try: + company = jobs.sources.add_by_hand(store, body.name, careers_url=body.careers_url, + notes=body.notes, hand_reason=body.hand_reason) + except jobs.sources.Conflict as exc: + raise HTTPException(409, str(exc)) from exc + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return _one_company(store, company.key) + + +class CompanyPatch(BaseModel): + name: str | None = None + careers_url: str | None = None + notes: str | None = None + hand_reason: HandReason | None = None + + +@router.patch("/companies/{key}") +def edit_company(key: str, body: CompanyPatch, store: StoreDep) -> CompanyOut: + """Change any of a company's name, careers link, notes and reason; a field + left out stays. The key never changes, and `name` only for a company + without a board: 409 for one with a subscription, shown or not (see + `jobs.sources.edit_company`).""" + fields = body.model_dump(exclude_unset=True) + if any(v is None for v in fields.values()): + raise HTTPException(422, "A field is left out, not null.") + try: + jobs.sources.edit_company(store, key, **fields) + except KeyError: + raise HTTPException(404, f"no such company: {key}") from None + except jobs.sources.Conflict as exc: + raise HTTPException(409, str(exc)) from exc + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return _one_company(store, key) + + +@router.post("/companies/{key}/checked") +def company_checked(key: str, store: StoreDep) -> CompanyOut: + """A person has just looked at the company's careers page.""" + if store.company(key) is None: + raise HTTPException(404, f"no such company: {key}") + store.update_company(key, checked_at=store.now()) + return _one_company(store, key) + + +@router.delete("/companies/{key}", status_code=204) +def remove_company(key: str, store: StoreDep) -> Response: + """Delete a company and every agent's stance on it. 409 for one with a + board shown or with open roles; detach the board first.""" + try: + jobs.sources.remove_company(store, key) + except KeyError: + raise HTTPException(404, f"no such company: {key}") from None + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + return Response(status_code=204) + + class CompanyBoardBody(BaseModel): board: str diff --git a/tests/test_web.py b/tests/test_web.py index 975b72a..2a6066f 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -421,24 +421,27 @@ def test_companies_lists_board_and_open_role_count( store.add_company("Acme", key="acme", ats="greenhouse", country="DK") store.upsert_role(posting("1", company="Acme"), sub) store.add_company("Other", key="other", domain="other.example") # no board found - store.add_company("Novo", key="novo", ats="successfactors", # followed, not readable + store.add_company("Novo", key="novo", ats="taleo", # followed, not readable careers_url="https://novo.example/careers") res = client.get("/api/companies") assert res.status_code == 200 [row] = [c for c in res.json() if c["key"] == "acme"] - assert row == {"key": "acme", "name": "Acme", "board": "greenhouse", - "subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1, - "ats": "greenhouse", "ats_slug": "", "careers_url": "", - "country": "DK", "failing_since": "", "failing_reason": "", - "from_search": False} + assert row == {"key": "acme", "name": "Acme", "domain": "", "notes": "", + "board": "greenhouse", "subscription_id": sub.id, "on": True, + "affinity": 0, "open_roles": 1, "ats": "greenhouse", "ats_slug": "", + "careers_url": "", "country": "DK", "failing_since": "", + "failing_reason": "", "from_search": False, "hand_reason": "", + "checked_at": "", "by_hand": False} [other] = [c for c in res.json() if c["key"] == "other"] assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( "", None, None, "") [novo] = [c for c in res.json() if c["key"] == "novo"] assert (novo["board"], novo["subscription_id"], novo["ats"], novo["careers_url"]) == ( - "", None, "successfactors", "https://novo.example/careers") + "", None, "taleo", "https://novo.example/careers") assert not any(c["from_search"] for c in res.json()) + # Companies with no board shown, unread or not, are the list checked by hand. + assert {c["key"] for c in res.json() if c["by_hand"]} == {"other", "novo"} def test_judging_a_company_known_only_from_a_search_creates_its_row( @@ -1214,10 +1217,152 @@ def test_a_companys_board_is_set_replaced_and_detached( assert client.post("/api/companies/nope/board", json={"board": "lever:acme"}).status_code == 404 assert client.delete("/api/companies/nope/board").status_code == 404 - bad = client.post("/api/companies/acme/board", json={"board": "workday:acme"}) + bad = client.post("/api/companies/acme/board", json={"board": "taleo:acme"}) assert bad.status_code == 422 and "host:slug" in bad.json()["detail"] +def test_a_company_is_added_by_hand_with_a_reason_and_a_link( + client: TestClient, store: Store) -> None: + res = client.post("/api/companies", json={ + "name": "Revolut Ltd", "careers_url": " https://revolut.example/careers ", + "notes": "wants a login"}) + assert res.status_code == 201, res.text + assert res.json() == { + "key": "revolut", "name": "Revolut Ltd", "domain": "", "notes": "wants a login", + "board": "", "subscription_id": None, "on": None, "affinity": 0, "open_roles": 0, + "ats": "", "ats_slug": "", "careers_url": "https://revolut.example/careers", + "country": "", "failing_since": "", "failing_reason": "", "from_search": False, + "hand_reason": "added", "checked_at": "", "by_hand": True} + assert store.company("revolut").hand_reason == "added" # type: ignore[union-attr] + + other = client.post("/api/companies", json={"name": "Meta", "hand_reason": "blocked"}) + assert other.status_code == 201 and other.json()["hand_reason"] == "blocked" + # The key comes from the name, so a legal suffix or case is the same company. + assert client.post("/api/companies", json={"name": "REVOLUT"}).status_code == 409 + assert store.company("revolut").notes == "wants a login" # type: ignore[union-attr] + + +def test_adding_a_company_by_hand_refuses_a_bad_link_reason_or_name( + client: TestClient, store: Store) -> None: + for link in ("javascript:alert(1)", "ftp://x.example/jobs", "revolut.example/careers", + "https://"): + res = client.post("/api/companies", json={"name": "Revolut", "careers_url": link}) + assert res.status_code == 422, link + assert client.post("/api/companies", json={"name": "Revolut", "careers_url": ""} + ).status_code == 201 + assert client.post("/api/companies", json={"name": "Meta", "hand_reason": "lazy"} + ).status_code == 422 + assert client.post("/api/companies", json={"name": " "}).status_code == 422 + assert client.post("/api/companies", json={}).status_code == 422 + assert [c.key for c in store.companies()] == ["revolut"] + + +def test_a_company_from_a_search_only_is_no_longer_one_once_it_has_a_reason( + client: TestClient, store: Store) -> None: + store.save_agent("A") + search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex") + item = posting("1", company="Novo") + item.source = "search" + store.upsert_role(item, search) + client.post("/api/companies/novo/affinity", json={"affinity": 1}) + [row] = client.get("/api/companies").json() + assert (row["from_search"], row["by_hand"]) == (True, False) + + res = client.patch("/api/companies/novo", json={"hand_reason": "unread"}) + assert res.status_code == 200 + assert (res.json()["from_search"], res.json()["by_hand"]) == (False, True) + + +def test_a_company_is_edited_and_only_the_fields_sent_change( + client: TestClient, store: Store) -> None: + store.add_company("Acme", key="acme", careers_url="https://acme.example/jobs", + notes="old", hand_reason="unread") + res = client.patch("/api/companies/acme", json={"notes": "new", "name": "Acme Labs"}) + assert res.status_code == 200, res.text + body = res.json() + assert (body["key"], body["name"], body["notes"], body["careers_url"], + body["hand_reason"]) == ("acme", "Acme Labs", "new", "https://acme.example/jobs", + "unread") + # A link and a reason can be cleared, since "" is a value. + res = client.patch("/api/companies/acme", json={"careers_url": "", "hand_reason": ""}) + assert (res.json()["careers_url"], res.json()["hand_reason"]) == ("", "") + assert client.patch("/api/companies/acme", json={}).status_code == 200 + + assert client.patch("/api/companies/acme", json={"careers_url": "javascript:x"} + ).status_code == 422 + assert client.patch("/api/companies/acme", json={"hand_reason": "lazy"}).status_code == 422 + assert client.patch("/api/companies/acme", json={"name": " "}).status_code == 422 + assert client.patch("/api/companies/acme", json={"notes": None}).status_code == 422 + assert client.patch("/api/companies/nope", json={"notes": "x"}).status_code == 404 + assert store.company("acme").name == "Acme Labs" # type: ignore[union-attr] + + +def test_a_company_with_a_board_keeps_its_name_and_takes_the_other_edits( + client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + import plugins.ats as ats + + monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) + store.add_company("Acme", key="acme") + assert client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"} + ).status_code == 200 + + res = client.patch("/api/companies/acme", json={"name": "Acme Labs"}) + assert res.status_code == 409 and "detach" in res.json()["detail"] + assert client.patch("/api/companies/acme", json={"name": "Acme"}).status_code == 200 + assert client.patch("/api/companies/acme", json={"notes": "hi"}).status_code == 200 + # Detached, the subscription is still there and the name stays fixed. + client.delete("/api/companies/acme/board") + assert client.patch("/api/companies/acme", json={"name": "Acme Labs"}).status_code == 409 + assert store.company("acme").name == "Acme" # type: ignore[union-attr] + + +def test_checking_a_company_stamps_now(client: TestClient, store: Store) -> None: + clock = Clock() + store.clock = clock + store.add_company("Acme", key="acme", hand_reason="unread") + res = client.post("/api/companies/acme/checked") + assert res.status_code == 200 + assert res.json()["checked_at"] == "2026-09-23T09:00:00+00:00" + clock.tick(48) + assert client.post("/api/companies/acme/checked").json()["checked_at"] == ( + "2026-09-25T09:00:00+00:00") + assert client.post("/api/companies/nope/checked").status_code == 404 + + +def test_a_company_is_deleted_unless_it_is_read( + client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + import plugins.ats as ats + + monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) + agent = store.save_agent("A") + store.add_company("Meta", key="meta", hand_reason="blocked") + store.set_affinity(agent.id, "meta", 1) + res = client.delete("/api/companies/meta") + assert res.status_code == 204 and res.content == b"" + assert store.company("meta") is None + assert store.company("meta", agent.id) is None + assert client.delete("/api/companies/meta").status_code == 404 + assert "meta" not in {c["key"] for c in client.get("/api/companies").json()} + + # A board shown: refused until it is detached; a detached board is no board. + store.add_company("Acme", key="acme") + sub_id = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"} + ).json()["source"]["id"] + res = client.delete("/api/companies/acme") + assert res.status_code == 409 and "greenhouse:acme" in res.json()["detail"] + store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id)) + client.delete("/api/companies/acme/board") + assert client.delete("/api/companies/acme").status_code == 204 + + # Open roles hold a company even where no board is on: they would be + # orphaned from the list. + store.add_company("Beta", key="beta") + store.upsert_role(posting("2", company="Beta")) + res = client.delete("/api/companies/beta") + assert res.status_code == 409 and "open roles" in res.json()["detail"] + assert store.company("beta") is not None + + def test_the_pull_a_source_switch_queues_is_refused_quietly_while_one_runs( app: FastAPI, store: Store) -> None: """The four actions that queue a `pull` (switching a source on, following From 161e89c9557ec1ffb6f88971ddd422253be734da Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 19:25:46 +0200 Subject: [PATCH 4/7] Sort the by-hand list by the Watchlist's careers_opened_at and print the watch rating Co-Authored-By: Claude Sonnet 5.5 --- app/cli.py | 18 +++++++++++------- app/jobs/sources.py | 6 +++--- tests/test_sources.py | 24 ++++++++++++++---------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/app/cli.py b/app/cli.py index 35b3cbe..fe92c5d 100644 --- a/app/cli.py +++ b/app/cli.py @@ -409,12 +409,14 @@ def edit_company(store: Store, args: argparse.Namespace) -> int: def company_checked(store: Store, key: str) -> int: - """`app companies checked acme`: its careers page was looked at just now.""" + """`app companies checked acme`: its careers page was looked at just now, + which the Watchlist keeps as `careers_opened_at`, the same stamp opening + the page from there sets.""" if store.company(key) is None: print(f"companies checked: no company {key!r}", file=sys.stderr) return 1 - company = store.update_company(key, checked_at=store.now()) - print(json.dumps({"key": key, "checked_at": company.checked_at})) + company = store.update_company(key, careers_opened_at=store.now()) + print(json.dumps({"key": key, "careers_opened_at": company.careers_opened_at})) return 0 @@ -437,14 +439,16 @@ def remove_company(store: Store, key: str) -> int: def list_by_hand(store: Store) -> int: """`app companies by-hand`: the companies with no board shown that are not - known from a search only, one line each, never checked first and then the - longest ago, as the Sources page lists them; see `jobs.sources.by_hand`.""" + known from a search only, one line each, never opened first and then the + longest ago; `watch` is the Watchlist's rating, yes, no or a dash for + unrated. See `jobs.sources.by_hand`.""" from app import jobs rows = jobs.sources.by_hand(store) for c in rows: - checked = c.checked_at[:10] or "never" - print(f"{c.name:30} {c.hand_reason or '-':8} {checked:10} {c.careers_url}") + watch = "-" if c.watch is None else "yes" if c.watch else "no" + opened = c.careers_opened_at[:10] or "never" + print(f"{c.name:30} {c.hand_reason or '-':8} {watch:4} {opened:10} {c.careers_url}") print(f"{len(rows)} by hand") return 0 diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 9e566b6..76ceb5e 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -169,13 +169,13 @@ def known_from_search_only(company: Company, ever_boarded: set[str]) -> bool: def by_hand(store: Store) -> list[Company]: """The list checked by hand: companies with no board shown that are not - known from a search only. Never checked first, then the longest ago; the - name breaks a tie.""" + known from a search only. Never opened first, then the longest ago + (`careers_opened_at`, the Watchlist's stamp); the name breaks a tie.""" boards = store.company_boards() ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} rows = [c for c in store.companies() if c.key not in boards and not known_from_search_only(c, ever_boarded)] - return sorted(rows, key=lambda c: (c.checked_at, c.name.casefold())) + return sorted(rows, key=lambda c: (c.careers_opened_at, c.name.casefold())) class Conflict(ValueError): diff --git a/tests/test_sources.py b/tests/test_sources.py index dfedad0..d98bced 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -373,14 +373,14 @@ def test_remove_company_refuses_a_board_shown_and_open_roles( sources.remove_company(store, "open") -def test_by_hand_lists_the_unboarded_never_checked_first( +def test_by_hand_lists_the_unboarded_never_opened_first( store: Store, boards: Boards) -> None: sources.add_by_hand(store, "Zeta") sources.add_by_hand(store, "Beta") sources.add_by_hand(store, "Old") sources.add_by_hand(store, "Recent") - store.update_company("old", checked_at="2026-08-01T00:00:00+00:00") - store.update_company("recent", checked_at="2026-09-20T00:00:00+00:00") + store.update_company("old", careers_opened_at="2026-08-01T00:00:00+00:00") + store.update_company("recent", careers_opened_at="2026-09-20T00:00:00+00:00") sources.set_board(store, "read", "greenhouse:acme", name="Read") # a board: not listed store.add_company("Pleo", key="pleo", ats="teamtailor") # unread, no reason: listed store.add_company("Quiet", key="quiet") # from a search only @@ -486,8 +486,8 @@ def test_cli_edit_checked_and_remove_a_company_by_hand( assert "no company 'nobody'" in capsys.readouterr().err assert _cli(tmp_path, "checked", "acme") == 0 - stamp = json.loads(capsys.readouterr().out)["checked_at"] - assert stamp and store.company("acme").checked_at == stamp # type: ignore[union-attr] + stamp = json.loads(capsys.readouterr().out)["careers_opened_at"] + assert stamp and store.company("acme").careers_opened_at == stamp # type: ignore[union-attr] assert _cli(tmp_path, "checked", "nobody") == 1 capsys.readouterr() @@ -499,20 +499,24 @@ def test_cli_edit_checked_and_remove_a_company_by_hand( assert _cli(tmp_path, "remove", "acme") == 1 -def test_cli_by_hand_prints_never_checked_first_then_the_oldest_check( +def test_cli_by_hand_prints_never_opened_first_then_the_oldest_and_the_watch_rating( store: Store, boards: Boards, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: store.add_company("Recent", careers_url="https://recent.example/jobs", - hand_reason="gone", checked_at="2026-09-20T08:00:00+00:00") - store.add_company("Old", hand_reason="robots", checked_at="2026-08-01T08:00:00+00:00") + hand_reason="gone") + store.add_company("Old", hand_reason="robots") store.add_company("Never", careers_url="https://never.example/jobs", hand_reason="added") + store.update_company("recent", careers_opened_at="2026-09-20T08:00:00+00:00", watch=0) + store.update_company("old", careers_opened_at="2026-08-01T08:00:00+00:00", watch=1) sources.set_board(store, "read", "greenhouse:acme", name="Read") assert _cli(tmp_path, "by-hand") == 0 lines = capsys.readouterr().out.splitlines() assert [ln.split()[0] for ln in lines[:-1]] == ["Never", "Old", "Recent"] - assert lines[0].split() == ["Never", "added", "never", "https://never.example/jobs"] - assert lines[1].split() == ["Old", "robots", "2026-08-01"] + assert lines[0].split() == ["Never", "added", "-", "never", "https://never.example/jobs"] + assert lines[1].split() == ["Old", "robots", "yes", "2026-08-01"] + assert lines[2].split() == ["Recent", "gone", "no", "2026-09-20", + "https://recent.example/jobs"] assert lines[-1] == "3 by hand" From 1ebfab1b997d620ad241388c0147845efe63e422 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 19:26:08 +0200 Subject: [PATCH 5/7] Bring the by-hand plan in line with the Watchlist tab and migration 16 Co-Authored-By: Claude Sonnet 5.5 --- docs/plan-hosts-and-by-hand.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/plan-hosts-and-by-hand.md b/docs/plan-hosts-and-by-hand.md index e5c5cf3..fd478f6 100644 --- a/docs/plan-hosts-and-by-hand.md +++ b/docs/plan-hosts-and-by-hand.md @@ -38,7 +38,7 @@ The 49 were missed because the slug is not the company's name (`BoschGroup`, `ji **No reader is built on a key lifted from a site's script.** Maersk's API, New Work's and Welcome to the Jungle's search index answer only with a key that the page's JavaScript carries. They go on the by-hand list. -**The by-hand list is the `companies` table.** A company is on it when it has no shown board and is not known from a search only. `careers_url` and `notes` exist already; migration 15 adds `hand_reason` and `checked_at`. No second table, since a company has one careers link and the list needs no history of checks. +**The by-hand list is the `companies` table, and the Watchlist tab is the list.** A company is on it when it has no shown board and is not known from a search only. `careers_url`, `notes`, `watch` (the rating) and `careers_opened_at` exist already, the last being the date a person last looked at the careers page; migration 16 adds `hand_reason`. There is no `checked_at`, since opening the page from the Watchlist stamps the date already. No second table, since a company has one careers link and the list needs no history of checks. **A link typed by hand is kept.** `add_discovered` and the seed fill `careers_url` only when it is empty. @@ -75,16 +75,15 @@ def probe_ok(body: bytes) -> bool: ... Slugs: Workday `lego.wd103/LEGO_External`; SuccessFactors, Phenom and the sitemap reader take the site's hostname, with a path where the site has one (`jobs.teradyne.com/MiR`); Oracle `/`. -Company JSON, in `GET /api/companies`: `CompanyOut` gains `domain`, `notes`, `hand_reason`, `checked_at` and `by_hand` (bool). `hand_reason` is one of `''`, `unread` (readable, no parser yet), `robots`, `blocked` (the site answers a program with a challenge), `key`, `group`, `gone`, `added` (by hand or by an import, not looked at). `from_search` is false for a company with a `hand_reason`. +Company JSON, in `GET /api/companies`: `CompanyOut` has `domain`, `city`, `watch` and `careers_opened_at` from the Watchlist, and gains `notes`, `hand_reason` and `by_hand` (bool, no board shown and not `from_search`). `hand_reason` is one of `''`, `unread` (readable, no parser yet), `robots`, `blocked` (the site answers a program with a challenge), `key`, `group`, `gone`, `added` (by hand or by an import, not looked at). `from_search` is false for a company with a `hand_reason`. | Endpoint | Body | Answer | |---|---|---| | `POST /api/companies` | `name`, `careers_url`, `notes`, `hand_reason` (default `added`) | 201 `CompanyOut`; 409 when the key is taken | | `PATCH /api/companies/{key}` | any of `name`, `careers_url`, `notes`, `hand_reason` | `CompanyOut`; the key stays; 409 on `name` for a company with a board | -| `POST /api/companies/{key}/checked` | none | `CompanyOut` with `checked_at` now | | `DELETE /api/companies/{key}` | none | 204; 409 when it has a board or open roles; its `agent_companies` rows go with it | -`careers_url` is http or https, as `safeUrl` demands on the page. +`POST /api/companies/{key}/careers` and `POST /api/companies/{key}/opened` of the Watchlist stay and answer a `Company`; `POST .../careers` and `PATCH` check the link with the same function (`check_careers_url` in `app/jobs/sources.py`). `careers_url` is http or https, as `safeUrl` demands on the page. A person looking at the page is `POST .../opened`, which stamps `careers_opened_at`. ## Work packages @@ -94,7 +93,7 @@ Wave 0, in parallel, from today's `main`: |---|---|---|---| | 0 Host modules | `wp0-hosts` | Opus | `plugins/ats.py`, `plugins/atshosts/`, `app/jobs/discover.py`, `tests/test_ats.py`, `tests/test_discover.py`, `tests/test_atshosts.py` | | 1 By-hand backend | `wp1-byhand-api` | Sonnet | `app/models.py`, `app/storage/schema.py`, `app/storage/store.py`, `app/web/api.py`, `app/jobs/sources.py`, `app/cli.py`, `tests/test_storage.py`, `tests/test_web.py`, `tests/test_sources.py` | -| 2 By-hand page | `wp2-byhand-ui` | Opus | `app/templates/settings/sources.html`, `app/static/byhand.js`, `app/static/companies.js`, `app/static/app.css`, `app/web/pages.py` | +| 2b By-hand editing | `wp2b-byhand-ui` | Opus | `app/static/watchlist.js`, `app/templates/watchlist.html`, `app/static/app.css`, `tests/test_pages.py` | Wave 1, in parallel, from `main` after WP0 is merged. Each owns its modules in `plugins/atshosts/`, `tests/test_ats_.py` and `tests/fixtures/ats/*`, and nothing else: @@ -111,9 +110,9 @@ Wave 2 is decided after wave 1 by the roles each company would add: a module per **WP0.** The eight hosts move into modules with no change in behaviour: the existing tests pass with edits to imports only. `careers_url`, `is_known_target`, `_allowed_host` and `_paged` lose their Teamtailor branches to the module. `_get` takes `headers`. `detail` and the robots check are added to the shared code with tests against a fake host. `discover.py` takes `READABLE`, `HOSTS`, `_PROBES` and the body checks of `probe` from the modules; the hosts it names but cannot read stay in a table of its own. Discovery gains Lever's EU host. The tests that use `workday:` or `successfactors` as the unreadable host use `taleo` instead. -**WP1.** Migration 15, the four endpoints, `CompanyOut`'s new fields, `remove_company` taking the `agent_companies` rows, `careers_url` filled only when empty in `add_discovered` and `seed_companies`, the seed reading `hand_reason`, and `app companies add`, `edit`, `checked` and `remove` in the CLI. The `workday:` and `successfactors` fixtures in its three test files become `taleo`. +**WP1.** Migration 16 with `hand_reason` only, the endpoints `POST /api/companies`, `PATCH` and `DELETE`, `CompanyOut`'s new fields, `remove_company` taking the `agent_companies` rows, `careers_url` filled only when empty in `add_discovered` and `seed_companies`, the seed reading `hand_reason`, `check_careers_url` as the one rule for a typed link, and `app companies add`, `edit`, `checked`, `by-hand` and `remove` in the CLI, where `checked` stamps `careers_opened_at` and `by-hand` lists the never opened first. The `workday:` and `successfactors` fixtures in its three test files become `taleo`. -**WP2.** A section "Check by hand" in the Sources rail, after Company boards: a count, a New company button, and rows sorted by `checked_at`, never checked first. A row shows the name, the affinity control, the reason, the careers link, the note, "checked 3 days ago", and Checked, Edit and Remove. New and Edit use the form pattern of the job boards section (`sources.js`: a `