From a466ea52b78a6d1cf8f3a9cae25c50993958f444 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:02:18 +0200 Subject: [PATCH 01/11] Give a company a by-hand reason and a date last checked, in migration 15 Co-Authored-By: Claude Sonnet 5.5 --- app/models.py | 16 +++++++++ app/storage/schema.py | 11 +++++- app/storage/store.py | 30 ++++++++++++---- tests/test_storage.py | 82 ++++++++++++++++++++++++++++++++++++------- 4 files changed, 120 insertions(+), 19 deletions(-) diff --git a/app/models.py b/app/models.py index 562ac85..420efc2 100644 --- a/app/models.py +++ b/app/models.py @@ -644,6 +644,16 @@ STATUSES: tuple[str, ...] = get_args(Status) #: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score). AFFINITY = range(-1, 3) +#: Why a company is on the list checked by hand. `unread` is a host the app +#: could read and has no parser for yet, `robots` a site whose robots.txt +#: disallows it, `blocked` one that answers a program with a challenge, `key` +#: one whose API answers only with a key lifted from its own script, `group` a +#: company that sits inside its owner's board, `gone` one whose site is dead, +#: and `added` one put there by hand or by an import and not looked at. '' +#: is none of these: the company is read, or is not a matter for the list. +HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"] +HAND_REASONS: tuple[str, ...] = get_args(HandReason) + # -------------------------------------------------------------- posting dates @@ -924,6 +934,12 @@ class Company(Model): #: The name `ats` knows the company by, the `acme` of acme.teamtailor.com; #: "" where discovery found none. Kept for a host no parser reads yet. ats_slug: str = "" + #: Why the company is read by hand, one of `HAND_REASONS`; "" when it is + #: not on that list. + hand_reason: str = "" + #: When a person last looked at its careers page, an ISO timestamp; "" + #: for never. + checked_at: str = "" created_at: str updated_at: str diff --git a/app/storage/schema.py b/app/storage/schema.py index cb887da..c63194d 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -107,6 +107,8 @@ CREATE TABLE IF NOT EXISTS role_tags ( -- yet. country is an ISO code and city the head office's. ats_slug is the -- name that host knows the company by (the acme of acme.teamtailor.com), -- '' where its URL carries none, kept for the parser written later. +-- hand_reason is why a person reads the careers page (models.HAND_REASONS), +-- '' when nobody has to, and checked_at when one last did, '' for never. CREATE TABLE IF NOT EXISTS companies ( key TEXT PRIMARY KEY, name TEXT NOT NULL, @@ -119,7 +121,9 @@ CREATE TABLE IF NOT EXISTS companies ( ats TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', city TEXT NOT NULL DEFAULT '', - ats_slug TEXT NOT NULL DEFAULT '' + ats_slug TEXT NOT NULL DEFAULT '', + hand_reason TEXT NOT NULL DEFAULT '', + checked_at TEXT NOT NULL DEFAULT '' ); -- The targets desk's runner hands to plugins. newsdesk's shape, with its two @@ -423,6 +427,11 @@ ALTER TABLE agents DROP COLUMN intent; ALTER TABLE agents DROP COLUMN intent_key; ALTER TABLE agents DROP COLUMN intent_vec; ALTER TABLE scores ADD COLUMN prompt TEXT NOT NULL DEFAULT ''; +"""), + (15, "companies.hand_reason and checked_at: the companies read by hand are a list " + "with a reason and a date last looked at", """ +ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; +ALTER TABLE companies ADD COLUMN checked_at TEXT NOT NULL DEFAULT ''; """), ) diff --git a/app/storage/store.py b/app/storage/store.py index 1f7d22d..5a5adea 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -21,7 +21,7 @@ import numpy as np from desk.models import Item from app.models import ( - AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, + AFFINITY, HAND_REASONS, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, Subscription, Weights, check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key, @@ -94,7 +94,7 @@ _EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)" _EXCERPT_RAW = 240 _COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city", - "ats_slug") + "ats_slug", "hand_reason", "checked_at") #: A company as the store serves it: the row and the asked-for agent's affinity. _COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c @@ -1162,18 +1162,22 @@ class Store: def add_company(self, name: str, *, key: str = "", domain: str = "", status: str = "researching", careers_url: str = "", notes: str = "", ats: str = "", country: str = "", - city: str = "", ats_slug: str = "") -> Company: - """Raises ValueError on a key already taken.""" + city: str = "", ats_slug: str = "", hand_reason: str = "", + checked_at: str = "") -> Company: + """Raises ValueError on a key already taken or a `hand_reason` outside + `HAND_REASONS`.""" name = name.strip() key = key or company_key(name) if not key: raise ValueError("a company needs a name") + _check_hand_reason(hand_reason) try: with self.transaction() as conn: self._insert_company(conn, key, name, { "domain": domain, "status": status, "careers_url": careers_url, "notes": notes, "ats": ats, "country": country, "city": city, - "ats_slug": ats_slug}) + "ats_slug": ats_slug, "hand_reason": hand_reason, + "checked_at": checked_at}) except sqlite3.IntegrityError as exc: raise ValueError(f"company {key!r} already exists") from exc return self._company(key) @@ -1237,6 +1241,8 @@ class Store: unknown = set(fields) - set(_COMPANY_FIELDS) if unknown: raise ValueError(f"not a company field: {', '.join(sorted(unknown))}") + if "hand_reason" in fields: + _check_hand_reason(fields["hand_reason"]) if fields: sets = ", ".join(f"{f}=?" for f in fields) changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?", @@ -1246,7 +1252,13 @@ class Store: return self._company(key) def remove_company(self, key: str) -> bool: - return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0 + """Delete the row and every agent's stance on it, in one transaction: + `agent_companies` holds the key with no foreign key to hold it back, so + a row left behind would give a company added again under the same name + an affinity it never got. Roles and subscriptions are not touched.""" + with self.transaction() as conn: + conn.execute("DELETE FROM agent_companies WHERE company_key=?", (key,)) + return conn.execute("DELETE FROM companies WHERE key=?", (key,)).rowcount > 0 def _company(self, key: str, agent_id: int | None = None) -> Company: found = self.company(key, agent_id) @@ -1541,6 +1553,12 @@ def _check_status(status: str) -> None: f"({', '.join(STATUSES)})") +def _check_hand_reason(reason: Any) -> None: + if reason not in HAND_REASONS: + raise ValueError(f"hand_reason is one of {', '.join(repr(r) for r in HAND_REASONS)}, " + f"not {reason!r}") + + def _check_affinity(affinity: Any) -> None: if isinstance(affinity, bool) or not isinstance(affinity, int) or affinity not in AFFINITY: raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}") diff --git a/tests/test_storage.py b/tests/test_storage.py index 210fb36..0e13647 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -69,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 14 + assert LATEST == 15 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -88,10 +88,18 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]: return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")] +def _undo_v15(conn: sqlite3.Connection) -> None: + """A fresh file put back to the v14 shape, without the by-hand columns of + a company, and then to the v13 one by `_undo_v14`. Every downgrade below + starts here, since v15 is the last migration.""" + conn.execute("ALTER TABLE companies DROP COLUMN hand_reason") + conn.execute("ALTER TABLE companies DROP COLUMN checked_at") + _undo_v14(conn) + + def _undo_v14(conn: sqlite3.Connection) -> None: """A fresh file put back to the v13 shape: an agent's intent and its - cached vector on the agent row, no prompts, no prompt on a score. Every - downgrade below starts here, since v14 is the last migration.""" + cached vector on the agent row, no prompts, no prompt on a score.""" conn.execute("DROP TABLE agent_prompts") conn.execute("ALTER TABLE scores DROP COLUMN prompt") conn.execute("ALTER TABLE agents ADD COLUMN intent TEXT NOT NULL DEFAULT ''") @@ -104,7 +112,7 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None: Store(fresh) Store(old) with sqlite3.connect(old) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at for column in ("permit", "languages", "extract_tries", "summary", "summarised_at", "article_text", "article_at", "read_at", "posted_at", "salary_text"): @@ -131,7 +139,7 @@ def test_a_v8_file_gains_ats_slug_and_keeps_its_companies(tmp_path: Path) -> Non path = tmp_path / "v8.db" Store(path).add_company("Acme", ats="teamtailor", country="DK") with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("ALTER TABLE companies DROP COLUMN ats_slug") conn.execute("DROP INDEX idx_roles_posted") # v10, which follows conn.execute("ALTER TABLE roles DROP COLUMN posted_at") @@ -150,7 +158,7 @@ def _downgrade_to_v4(path: Path) -> None: """A fresh file put back to the v4 shape: votes on the role, affinity on the company, no read state, no tags, no per-agent tables.""" with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("ALTER TABLE roles DROP COLUMN read_at") conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9 @@ -262,7 +270,7 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( assert before_trusted is not None and before_untrusted is not None with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("DROP INDEX idx_roles_posted") conn.execute("ALTER TABLE roles DROP COLUMN posted_at") conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12 @@ -290,7 +298,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( for item in (wrong, several, given, extracted): store.upsert_role(item) with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,)) conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", @@ -317,7 +325,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P store.replace_scores(backend.id, [(uid, 0.7, "")]) vec = np.asarray([0.6, 0.8], dtype=np.float32) with sqlite3.connect(path) as conn: - _undo_v14(conn) + _undo_v15(conn) conn.execute("UPDATE agents SET intent = 'python backend', intent_key = 'k', " "intent_vec = ? WHERE id = ?", (pack(vec), backend.id)) conn.execute("UPDATE agents SET intent = ' \n ' WHERE id = ?", (blank.id,)) @@ -325,7 +333,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14] + assert migrated.migrations_applied == [14, 15] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -339,6 +347,25 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P assert not {"intent", "intent_key", "intent_vec"} & columns +def test_v15_gives_a_company_no_reason_and_no_check_and_keeps_its_row(tmp_path: Path) -> None: + fresh, path = tmp_path / "fresh.db", tmp_path / "v14.db" + Store(fresh) + Store(path).add_company("Acme", careers_url="https://acme.example/jobs", notes="met them") + with sqlite3.connect(path) as conn: + conn.execute("ALTER TABLE companies DROP COLUMN hand_reason") + conn.execute("ALTER TABLE companies DROP COLUMN checked_at") + conn.execute("PRAGMA user_version = 14") + + migrated = Store(path) + + assert migrated.migrations_applied == [15] + acme = migrated.company("acme") + assert acme is not None + assert (acme.careers_url, acme.notes, acme.hand_reason, acme.checked_at) == ( + "https://acme.example/jobs", "met them", "", "") + assert sorted(_columns(path, "companies")) == sorted(_columns(fresh, "companies")) + + def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None: ai = Prompt(name="AI leadership", text="Head of AI") fde = Prompt(name="Forward deployed", text="FDE lead") @@ -424,13 +451,13 @@ def test_posted_host_reads_board_then_provider() -> None: def test_posted_at_trusts_only_a_known_host() -> None: - assert "greenhouse" in TRUSTED_POSTED_HOSTS and "workday" not in TRUSTED_POSTED_HOSTS + assert "greenhouse" in TRUSTED_POSTED_HOSTS and "taleo" not in TRUSTED_POSTED_HOSTS early = "2020-01-01T00:00:00+00:00" late = "2026-09-23T09:00:00+00:00" # A trusted host's earlier date wins over first_seen... assert posted_at(early, late, "greenhouse") == early # ...but an untrusted (or unknown) host never moves it off first_seen. - assert posted_at(early, late, "workday") == late + assert posted_at(early, late, "taleo") == late assert posted_at(early, late, "") == late @@ -982,6 +1009,37 @@ def test_companies_crud(store: Store) -> None: assert store.company("acme") is None +def test_a_company_takes_a_reason_from_the_vocabulary_and_a_check_date(store: Store) -> None: + acme = store.add_company("Acme", hand_reason="robots", checked_at="2026-09-01T00:00:00+00:00") + assert (acme.hand_reason, acme.checked_at) == ("robots", "2026-09-01T00:00:00+00:00") + assert store.add_company("Plain").hand_reason == "" + with pytest.raises(ValueError, match="hand_reason is one of"): + store.add_company("Odd", hand_reason="lazy") + assert store.company("odd") is None + assert store.update_company("acme", hand_reason="").hand_reason == "" + with pytest.raises(ValueError, match="hand_reason is one of"): + store.update_company("acme", hand_reason="lazy") + assert store.company("acme").hand_reason == "" # type: ignore[union-attr] + + +def test_removing_a_company_takes_every_agents_stance_on_it(store: Store) -> None: + store.add_company("Acme") + store.add_company("Other") + one, two = store.save_agent("One"), store.save_agent("Two") + store.set_affinity(one.id, "acme", 2) + store.set_affinity(two.id, "acme", -1) + store.set_affinity(one.id, "other", 1) + + assert store.remove_company("acme") is True + assert store.remove_company("acme") is False + + # Added again under the same name, it starts neutral for everyone. + store.add_company("Acme") + assert store.company("acme", one.id).affinity == 0 # type: ignore[union-attr] + assert store.company("acme", two.id).affinity == 0 # type: ignore[union-attr] + assert store.company("other", one.id).affinity == 1 # type: ignore[union-attr] + + def test_affinity_is_per_agent(store: Store) -> None: store.add_company("Acme", key="acme") store.add_company("Other", key="other") From b3cb93d5aa6b9cfbfb88c1ffc96283ef87dad43d Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:02:19 +0200 Subject: [PATCH 02/11] Keep a typed careers link, seed the by-hand reason and add the by-hand commands to app companies Co-Authored-By: Claude Sonnet 5.5 --- app/cli.py | 127 +++++++++++++++++++++-- app/jobs/sources.py | 125 +++++++++++++++++++++- tests/test_sources.py | 235 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 472 insertions(+), 15 deletions(-) diff --git a/app/cli.py b/app/cli.py index 6017af6..35b3cbe 100644 --- a/app/cli.py +++ b/app/cli.py @@ -3,7 +3,7 @@ app serve [--host H] [--port P] [--reload] app job pull|extract|embed|score [--payload JSON] app worker [--workers http=8,cpu=1] [--until-idle] - app companies seed|add|set-board|detach|discover|check + app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check app searches add|list|remove app agents list|prompt app stages [extract|summarise on|off] @@ -44,6 +44,8 @@ if str(_ROOT) not in sys.path: def build_parser() -> argparse.ArgumentParser: + from app.models import HAND_REASONS + parser = argparse.ArgumentParser( prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.") parser.add_argument("-c", "--config", default=None, @@ -71,10 +73,34 @@ def build_parser() -> argparse.ArgumentParser: seed = companies_sub.add_parser( "seed", help="import seed/companies.json, and the boards it names for rows without one") seed.add_argument("--path", default="seed/companies.json") + # One word after `add` is a company for the list checked by hand, two are + # a board and its company, which is what `add` meant before that list. add = companies_sub.add_parser( - "add", help="follow a company by its board, attaching or replacing an existing one's") - add.add_argument("board", help="host:slug, e.g. greenhouse:acme") - add.add_argument("name") + "add", help="add NAME to the list checked by hand, or with BOARD NAME follow a " + "company by its board, attaching or replacing an existing one's") + add.add_argument("first", metavar="NAME|BOARD", + help="the company, or host:slug, e.g. greenhouse:acme, before its name") + add.add_argument("second", metavar="NAME", nargs="?") + add.add_argument("--careers-url", default="", help="its careers page, http or https") + add.add_argument("--note", default="") + add.add_argument("--reason", default="added", choices=HAND_REASONS, metavar="REASON", + help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}") + edit = companies_sub.add_parser("edit", help="change a company's name, link, note or reason") + edit.add_argument("key") + edit.add_argument("--name") + edit.add_argument("--careers-url") + edit.add_argument("--note") + edit.add_argument("--reason", choices=HAND_REASONS, metavar="REASON", + help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}") + checked = companies_sub.add_parser( + "checked", help="record that its careers page was looked at just now") + checked.add_argument("key") + remove = companies_sub.add_parser( + "remove", help="delete a company with no board on and no open roles") + remove.add_argument("key") + companies_sub.add_parser( + "by-hand", help="the companies checked by hand, never checked first, then the " + "longest ago") set_board = companies_sub.add_parser( "set-board", help="attach a board to a company, or replace the one it has") set_board.add_argument("key", help="the company key, e.g. trade-republic") @@ -274,7 +300,15 @@ def run_companies(args: argparse.Namespace) -> int: if args.action == "seed": return seed_companies(store, args.path) if args.action == "add": - return add_company(store, args.board, args.name) + return add_company(store, args) + if args.action == "edit": + return edit_company(store, args) + if args.action == "checked": + return company_checked(store, args.key) + if args.action == "remove": + return remove_company(store, args.key) + if args.action == "by-hand": + return list_by_hand(store) if args.action == "set-board": return set_board(store, args.key, args.board) if args.action == "detach": @@ -318,7 +352,25 @@ def discover_company(store: Store, name: str, domain: str, *, add: bool) -> int: return 0 -def add_company(store: Store, board: str, name: str) -> int: +def add_company(store: Store, args: argparse.Namespace) -> int: + """`app companies add "Acme" [--careers-url URL] [--note TEXT] [--reason + REASON]` puts a company with no board on the list checked by hand; + `app companies add greenhouse:acme "Acme"` follows it by its board.""" + if args.second is not None: + return add_company_by_board(store, args.first, args.second) + from app import jobs + + try: + company = jobs.sources.add_by_hand(store, args.first, careers_url=args.careers_url, + notes=args.note, hand_reason=args.reason) + except ValueError as exc: + print(f"companies add: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": company.key, "hand_reason": company.hand_reason})) + return 0 + + +def add_company_by_board(store: Store, board: str, name: str) -> int: """`app companies add greenhouse:acme "Acme"`: fetch the board first and refuse if it does not answer or is not a known host. A company that exists gets the board attached or replaced. Port of seedgen's `-add`.""" @@ -334,6 +386,69 @@ def add_company(store: Store, board: str, name: str) -> int: return 0 +def edit_company(store: Store, args: argparse.Namespace) -> int: + """`app companies edit acme --careers-url https://acme.example/jobs`: only + the options given change; see `jobs.sources.edit_company`.""" + from app import jobs + + given = {"name": args.name, "careers_url": args.careers_url, "notes": args.note, + "hand_reason": args.reason} + try: + company = jobs.sources.edit_company( + store, args.key, **{f: v for f, v in given.items() if v is not None}) + except KeyError: + print(f"companies edit: no company {args.key!r}", file=sys.stderr) + return 1 + except ValueError as exc: + print(f"companies edit: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": company.key, "name": company.name, + "careers_url": company.careers_url, "notes": company.notes, + "hand_reason": company.hand_reason})) + return 0 + + +def company_checked(store: Store, key: str) -> int: + """`app companies checked acme`: its careers page was looked at just now.""" + if store.company(key) is None: + print(f"companies checked: no company {key!r}", file=sys.stderr) + return 1 + company = store.update_company(key, checked_at=store.now()) + print(json.dumps({"key": key, "checked_at": company.checked_at})) + return 0 + + +def remove_company(store: Store, key: str) -> int: + """`app companies remove acme`: refused for a company still read, see + `jobs.sources.remove_company`.""" + from app import jobs + + try: + jobs.sources.remove_company(store, key) + except KeyError: + print(f"companies remove: no company {key!r}", file=sys.stderr) + return 1 + except ValueError as exc: + print(f"companies remove: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": key, "removed": True})) + return 0 + + +def list_by_hand(store: Store) -> int: + """`app companies by-hand`: the companies with no board shown that are not + known from a search only, one line each, never checked first and then the + longest ago, as the Sources page lists them; see `jobs.sources.by_hand`.""" + from app import jobs + + rows = jobs.sources.by_hand(store) + for c in rows: + checked = c.checked_at[:10] or "never" + print(f"{c.name:30} {c.hand_reason or '-':8} {checked:10} {c.careers_url}") + print(f"{len(rows)} by hand") + return 0 + + def set_board(store: Store, key: str, board: str) -> int: """`app companies set-board trade-republic greenhouse:traderepublic`: attach the board to an existing company or replace the one it has.""" diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 4eedbea..9e566b6 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -15,9 +15,10 @@ import json import os from pathlib import Path from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit from app.config import Config -from app.models import Subscription, company_key +from app.models import HAND_REASONS, Company, Subscription, company_key from app.storage import Store from app.jobs.pull import FAILING, search_target @@ -150,12 +151,116 @@ def add_discovered(store: Store, name: str, domain: str, if sub is not None and sub.show: raise ValueError(f"{name} is read from {sub.target} and discovery found no " "readable board; nothing changed") + # A link somebody typed is kept: discovery's is only a guess at the page. store.update_company(key, ats=found.ats, ats_slug=found.ats_slug, - careers_url=found.careers_url or company.careers_url, + careers_url=company.careers_url or found.careers_url, **({"domain": domain} if not company.domain else {})) return key, 0 +def known_from_search_only(company: Company, ever_boarded: set[str]) -> bool: + """A row that exists only because a search listed its roles and somebody + judged it (`Store.set_affinity`): no board, not even a detached one + (`ever_boarded`, the keys of every `ats` subscription), and none of the + facts every way of following a company or listing it by hand writes.""" + return not (company.key in ever_boarded or company.ats or company.careers_url + or company.domain or company.hand_reason) + + +def by_hand(store: Store) -> list[Company]: + """The list checked by hand: companies with no board shown that are not + known from a search only. Never checked first, then the longest ago; the + name breaks a tie.""" + boards = store.company_boards() + ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} + rows = [c for c in store.companies() + if c.key not in boards and not known_from_search_only(c, ever_boarded)] + return sorted(rows, key=lambda c: (c.checked_at, c.name.casefold())) + + +class Conflict(ValueError): + """A change refused because of what the company already is, not because of + what was asked for: the pages answer it with 409, not 422.""" + + +def check_careers_url(url: str) -> str: + """`url` stripped; ValueError unless it is empty or an http(s) link. The + page puts it in an `href`, where `javascript:` would run.""" + url = url.strip() + if url: + parts = urlsplit(url) + if parts.scheme not in ("http", "https") or not parts.netloc: + raise ValueError("The careers link must start with http:// or https://.") + return url + + +def add_by_hand(store: Store, name: str, *, careers_url: str = "", notes: str = "", + hand_reason: str = "added") -> Company: + """A company with no board, put on the list checked by hand. Raises + Conflict when the key its name folds to is taken, ValueError for a name + with no letters, a link that is no http(s) URL or a reason outside + `HAND_REASONS`.""" + link = check_careers_url(careers_url) + key = company_key(name) + if key and store.company(key) is not None: + raise Conflict(f"{name.strip()} is on the list already as {key}.") + return store.add_company(name, key=key, careers_url=link, notes=notes, + hand_reason=hand_reason) + + +def edit_company(store: Store, key: str, **fields: str) -> Company: + """Change a company's `name`, `careers_url`, `notes` or `hand_reason`; the + key never changes. A new name is Conflict for a company with a board + subscription, shown or not, since the subscription is found by the key + its company name folds to and would be lost. Raises KeyError for an + unknown company and ValueError for a blank name, a link that is no http(s) + URL or a reason outside `HAND_REASONS`.""" + unknown = set(fields) - {"name", "careers_url", "notes", "hand_reason"} + if unknown: + raise ValueError(f"not a field to edit: {', '.join(sorted(unknown))}") + company = store.company(key) + if company is None: + raise KeyError(key) + if "careers_url" in fields: + fields["careers_url"] = check_careers_url(fields["careers_url"]) + if "name" in fields: + fields["name"] = fields["name"].strip() + if not fields["name"]: + raise ValueError("A company needs a name.") + if fields["name"] != company.name and store.company_subscription(key) is not None: + raise Conflict(f"{company.name} has a board, which is found by its name; " + "detach the board first.") + return store.update_company(key, **fields) + + +def remove_company(store: Store, key: str) -> None: + """Delete company `key` and every agent's stance on it. Raises KeyError + for an unknown company and Conflict for one still read: a company with + a board on (`show`, as `detach_board` decides) or with open roles, which + would otherwise vanish from the list with nothing said. A detached board + is no board, since nothing reads it and its roles are closed; its + subscription row stays, and follows the company if it comes back.""" + if store.company(key) is None: + raise KeyError(key) + sub = store.company_subscription(key) + if sub is not None and sub.show: + raise Conflict(f"{key} is read from {sub.target}; detach the board first") + if store.open_role_counts().get(key): + raise Conflict(f"{key} has open roles; detach its board first") + store.remove_company(key) + + +def _fill_empty(store: Store, key: str, **facts: str) -> None: + """Write each of `facts` to company `key` where the stored value is + empty: what a seed may add to a row that exists, since a person's link or + reason is never overwritten.""" + company = store.company(key) + assert company is not None + fill = {f: v for f, v in facts.items() if v and not getattr(company, f)} + if fill: + store.update_company(key, **fill) + + def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: """Import `seed/companies.json`: one `companies` row per entry, and its `board` through `set_board` where the entry names one. An entry without @@ -167,8 +272,10 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: existing row that never had a board gets the one the seed now names, so a seed update that gains boards reaches a database made before it. A row's board, once it has one (detached counts), and its affinity, notes and - status are never touched. The seed's boards are not fetched again: they - were read once when the seed was written, and `app companies check` + status are never touched; its `careers_url` and `hand_reason` are filled + where empty, so the seed can hand a row its link and its reason but never + overwrite the one a person set. The seed's boards are not fetched again: + they were read once when the seed was written, and `app companies check` finds the dead ones. Returns counts, and `problems`, one line per entry that could not land whole.""" from plugins.ats import is_known_target @@ -186,8 +293,15 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: board = "" facts = {f: str(row.get(f) or "") for f in ("domain", "country", "city", "careers_url", "ats", "ats_slug")} + reason = str(row.get("hand_reason") or "") + if reason not in HAND_REASONS: + problems.append(f"{key}: {reason!r} is not a hand_reason; kept without it") + reason = "" exists = store.company(key) is not None + if exists: + _fill_empty(store, key, hand_reason=reason) if exists and (not board or store.company_subscription(key) is not None): + _fill_empty(store, key, careers_url=facts["careers_url"]) skipped += 1 continue if board: @@ -200,13 +314,14 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: except ValueError as exc: problems.append(f"{key}: {exc}") if exists: + _fill_empty(store, key, careers_url=facts["careers_url"]) skipped += 1 continue # No board, or one that clashed: the company is still a row, and a # clashing board's host and slug are what discovery found. if board: facts["ats"], _, facts["ats_slug"] = board.partition(":") - store.add_company(name, key=key, **facts) + store.add_company(name, key=key, hand_reason=reason, **facts) added += 1 return {"added": added, "boards": boards, "skipped": skipped, "total": len(entries), "problems": problems} diff --git a/tests/test_sources.py b/tests/test_sources.py index 7665d3f..dfedad0 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -110,7 +110,7 @@ def test_a_board_that_does_not_answer_or_is_unknown_is_refused( with pytest.raises(ValueError, match="did not answer"): sources.set_board(store, "", "greenhouse:gone", name="Gone") with pytest.raises(ValueError, match="host:slug"): - sources.set_board(store, "", "workday:gone", name="Gone") + sources.set_board(store, "", "taleo:gone", name="Gone") with pytest.raises(ValueError, match="needs a name"): sources.set_board(store, "nobody", "greenhouse:acme") with pytest.raises(ValueError, match="is not the company"): @@ -161,20 +161,34 @@ def test_add_discovered_records_an_unreadable_host_with_its_slug( careers_url="https://acme.example/careers") assert sources.add_discovered(store, "Acme", "acme.example", found) == ("acme", 0) store.update_company("acme", notes="keep") - newer = Discovery(ats="workday", ats_slug="acme-inc", + newer = Discovery(ats="taleo", ats_slug="acme-inc", careers_url="https://acme.example/jobs") assert sources.add_discovered(store, "Acme", "acme.example", newer) == ("acme", 0) acme = store.company("acme") assert acme is not None + # The host and slug follow what discovery found last; the link, once + # there, is kept, since a person may have typed it. assert (acme.ats, acme.ats_slug, acme.careers_url, acme.notes) == ( - "workday", "acme-inc", "https://acme.example/jobs", "keep") + "taleo", "acme-inc", "https://acme.example/careers", "keep") + + +def test_add_discovered_fills_a_link_only_where_there_is_none( + store: Store, boards: Boards) -> None: + store.add_company("Acme", ats="teamtailor") + found = Discovery(ats="taleo", ats_slug="acme", careers_url="https://acme.example/jobs") + sources.add_discovered(store, "Acme", "acme.example", found) + assert store.company("acme").careers_url == "https://acme.example/jobs" # type: ignore[union-attr] + + store.update_company("acme", careers_url="https://typed.example/by-hand") + sources.add_discovered(store, "Acme", "acme.example", found) + assert store.company("acme").careers_url == "https://typed.example/by-hand" # type: ignore[union-attr] def test_add_discovered_keeps_a_board_when_nothing_readable_was_found( store: Store, boards: Boards) -> None: sources.set_board(store, "", "greenhouse:acme", name="Acme") with pytest.raises(ValueError, match="nothing changed"): - sources.add_discovered(store, "Acme", "acme.example", Discovery(ats="workday")) + sources.add_discovered(store, "Acme", "acme.example", Discovery(ats="taleo")) acme = store.company("acme") assert acme is not None and acme.ats == "greenhouse" @@ -240,6 +254,143 @@ def test_a_seed_board_that_clashes_still_lands_the_company( assert store.company_subscription("acme-two") is None +def test_a_seed_rerun_fills_an_empty_link_and_reason_and_never_overwrites_one( + store: Store, boards: Boards, tmp_path: Path) -> None: + store.add_company("Bare", key="bare") + store.add_company("Typed", key="typed", careers_url="https://typed.example/by-hand", + hand_reason="blocked", notes="mine") + seed = _seed(tmp_path, + {"name": "Bare", "careers_url": "https://bare.example/jobs", + "hand_reason": "unread"}, + {"name": "Typed", "careers_url": "https://seed.example/jobs", + "hand_reason": "robots"}, + {"name": "Fresh", "careers_url": "https://fresh.example/jobs", + "hand_reason": "key"}) + + result = sources.seed_companies(store, seed) + + assert (result["added"], result["skipped"], result["problems"]) == (1, 2, []) + got = {k: (c.careers_url, c.hand_reason, c.notes) for k in ("bare", "typed", "fresh") + if (c := store.company(k)) is not None} + assert got == { + "bare": ("https://bare.example/jobs", "unread", ""), + "typed": ("https://typed.example/by-hand", "blocked", "mine"), + "fresh": ("https://fresh.example/jobs", "key", "")} + # Run again, nothing moves. + assert sources.seed_companies(store, seed)["added"] == 0 + assert store.company("bare").hand_reason == "unread" # type: ignore[union-attr] + + +def test_a_seed_reason_outside_the_vocabulary_is_a_problem_not_a_row_value( + store: Store, boards: Boards, tmp_path: Path) -> None: + result = sources.seed_companies( + store, _seed(tmp_path, {"name": "Odd", "hand_reason": "lazy"})) + assert result["problems"] == ["odd: 'lazy' is not a hand_reason; kept without it"] + assert store.company("odd").hand_reason == "" # type: ignore[union-attr] + + +def test_a_seed_link_reaches_a_row_whose_board_clashes( + store: Store, boards: Boards, tmp_path: Path) -> None: + sources.set_board(store, "acme", "greenhouse:acme", name="Acme") + store.add_company("Twin", key="twin") + result = sources.seed_companies(store, _seed( + tmp_path, {"name": "Twin", "board": "greenhouse:acme", + "careers_url": "https://twin.example/jobs"})) + assert len(result["problems"]) == 1 + assert store.company("twin").careers_url == "https://twin.example/jobs" # type: ignore[union-attr] + + +# ------------------------------------------------------------- by hand + +def test_a_company_added_by_hand_has_no_board_and_a_checked_link(store: Store) -> None: + company = sources.add_by_hand(store, "Revolut Ltd", careers_url=" https://r.example/jobs ", + notes="login wall") + assert (company.key, company.hand_reason, company.careers_url, company.notes) == ( + "revolut", "added", "https://r.example/jobs", "login wall") + assert store.company_subscription("revolut") is None + with pytest.raises(sources.Conflict, match="on the list already"): + sources.add_by_hand(store, "REVOLUT") + for link in ("javascript:alert(1)", "r.example/jobs", "https://", "ftp://r.example"): + with pytest.raises(ValueError, match="http"): + sources.add_by_hand(store, "Other", careers_url=link) + with pytest.raises(ValueError, match="hand_reason"): + sources.add_by_hand(store, "Other", hand_reason="lazy") + with pytest.raises(ValueError, match="needs a name"): + sources.add_by_hand(store, " ") + assert [c.key for c in store.companies()] == ["revolut"] + + +def test_edit_company_changes_what_it_is_given_and_renames_only_without_a_board( + store: Store, boards: Boards) -> None: + sources.add_by_hand(store, "Acme", notes="old") + edited = sources.edit_company(store, "acme", name="Acme GmbH", hand_reason="gone") + assert (edited.key, edited.name, edited.notes, edited.hand_reason) == ( + "acme", "Acme GmbH", "old", "gone") + with pytest.raises(ValueError, match="needs a name"): + sources.edit_company(store, "acme", name=" ") + with pytest.raises(ValueError, match="not a field"): + sources.edit_company(store, "acme", ats="lever") + with pytest.raises(KeyError): + sources.edit_company(store, "nobody", notes="x") + + sources.set_board(store, "acme", "greenhouse:acme") + with pytest.raises(sources.Conflict, match="detach"): + sources.edit_company(store, "acme", name="Acme Labs") + # The name it has, and any other field, are no rename. + assert sources.edit_company(store, "acme", name="Acme GmbH", notes="new").notes == "new" + sources.detach_board(store, "acme") + with pytest.raises(sources.Conflict): + sources.edit_company(store, "acme", name="Acme Labs") + + +def test_remove_company_refuses_a_board_shown_and_open_roles( + store: Store, boards: Boards) -> None: + agent = store.save_agent("A") + sources.add_by_hand(store, "Hand") + store.set_affinity(agent.id, "hand", 2) + sources.remove_company(store, "hand") + assert store.company("hand") is None + assert store.company("hand", agent.id) is None + with pytest.raises(KeyError): + sources.remove_company(store, "hand") + + sub, _found, _closed = sources.set_board(store, "acme", "greenhouse:acme", name="Acme") + with pytest.raises(sources.Conflict, match="detach the board"): + sources.remove_company(store, "acme") + store.upsert_role(posting("1"), sub) + sources.detach_board(store, "acme") # closes the role: no board, none open + sources.remove_company(store, "acme") + assert store.company("acme") is None + # The detached subscription stays and is found again by a company of that name. + assert store.company_subscription("acme") is not None + + store.add_company("Open", key="open") + other = store.add_subscription("search", "jobindex:x", label="x", board="jobindex") + item = posting("2") + item.raw = {"company": "Open"} + store.upsert_role(item, other) + with pytest.raises(ValueError, match="open roles"): + sources.remove_company(store, "open") + + +def test_by_hand_lists_the_unboarded_never_checked_first( + store: Store, boards: Boards) -> None: + sources.add_by_hand(store, "Zeta") + sources.add_by_hand(store, "Beta") + sources.add_by_hand(store, "Old") + sources.add_by_hand(store, "Recent") + store.update_company("old", checked_at="2026-08-01T00:00:00+00:00") + store.update_company("recent", checked_at="2026-09-20T00:00:00+00:00") + sources.set_board(store, "read", "greenhouse:acme", name="Read") # a board: not listed + store.add_company("Pleo", key="pleo", ats="teamtailor") # unread, no reason: listed + store.add_company("Quiet", key="quiet") # from a search only + sources.set_board(store, "gone", "ashby:acme", name="Gone") + sources.detach_board(store, "gone") # detached: listed + + assert [c.key for c in sources.by_hand(store)] == [ + "beta", "gone", "pleo", "zeta", "old", "recent"] + + # -------------------------------------------------------------------- CLI def _cli(tmp_path: Path, *argv: str) -> int: @@ -289,6 +440,82 @@ def test_cli_seed_prints_counts_and_problems( assert "companies seed: odd: 'nosuch:odd' is not a known board" in captured.err +def test_cli_add_with_one_name_lists_a_company_by_hand_and_with_two_follows_a_board( + store: Store, boards: Boards, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + assert _cli(tmp_path, "add", "Revolut", "--careers-url", "https://r.example/jobs", + "--note", "login wall") == 0 + assert json.loads(capsys.readouterr().out) == {"key": "revolut", "hand_reason": "added"} + revolut = store.company("revolut") + assert revolut is not None + assert (revolut.careers_url, revolut.notes, revolut.hand_reason) == ( + "https://r.example/jobs", "login wall", "added") + assert _cli(tmp_path, "add", "Meta", "--reason", "blocked") == 0 + assert store.company("meta").hand_reason == "blocked" # type: ignore[union-attr] + capsys.readouterr() + + assert _cli(tmp_path, "add", "Revolut") == 1 + assert "on the list already" in capsys.readouterr().err + assert _cli(tmp_path, "add", "Odd", "--careers-url", "javascript:alert(1)") == 1 + assert "http" in capsys.readouterr().err + with pytest.raises(SystemExit): # argparse refuses the reason + _cli(tmp_path, "add", "Odd", "--reason", "lazy") + capsys.readouterr() + assert store.company("odd") is None + + # Two words are the board and the company, as before. + assert _cli(tmp_path, "add", "greenhouse:acme", "Acme") == 0 + assert json.loads(capsys.readouterr().out) == { + "key": "acme", "board": "greenhouse:acme", "roles": 3} + + +def test_cli_edit_checked_and_remove_a_company_by_hand( + store: Store, boards: Boards, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + store.add_company("Acme", key="acme", notes="keep", hand_reason="unread") + assert _cli(tmp_path, "edit", "acme", "--careers-url", "https://acme.example/jobs", + "--reason", "robots") == 0 + assert json.loads(capsys.readouterr().out) == { + "key": "acme", "name": "Acme", "careers_url": "https://acme.example/jobs", + "notes": "keep", "hand_reason": "robots"} + assert _cli(tmp_path, "edit", "acme", "--note", "") == 0 + assert store.company("acme").notes == "" # type: ignore[union-attr] + assert _cli(tmp_path, "edit", "acme", "--careers-url", "ftp://x") == 1 + assert "http" in capsys.readouterr().err + assert _cli(tmp_path, "edit", "nobody", "--note", "x") == 1 + assert "no company 'nobody'" in capsys.readouterr().err + + assert _cli(tmp_path, "checked", "acme") == 0 + stamp = json.loads(capsys.readouterr().out)["checked_at"] + assert stamp and store.company("acme").checked_at == stamp # type: ignore[union-attr] + assert _cli(tmp_path, "checked", "nobody") == 1 + capsys.readouterr() + + sources.set_board(store, "read", "greenhouse:acme", name="Read") + assert _cli(tmp_path, "remove", "read") == 1 + assert "detach the board first" in capsys.readouterr().err + assert _cli(tmp_path, "remove", "acme") == 0 + assert store.company("acme") is None + assert _cli(tmp_path, "remove", "acme") == 1 + + +def test_cli_by_hand_prints_never_checked_first_then_the_oldest_check( + store: Store, boards: Boards, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + store.add_company("Recent", careers_url="https://recent.example/jobs", + hand_reason="gone", checked_at="2026-09-20T08:00:00+00:00") + store.add_company("Old", hand_reason="robots", checked_at="2026-08-01T08:00:00+00:00") + store.add_company("Never", careers_url="https://never.example/jobs", hand_reason="added") + sources.set_board(store, "read", "greenhouse:acme", name="Read") + + assert _cli(tmp_path, "by-hand") == 0 + lines = capsys.readouterr().out.splitlines() + assert [ln.split()[0] for ln in lines[:-1]] == ["Never", "Old", "Recent"] + assert lines[0].split() == ["Never", "added", "never", "https://never.example/jobs"] + assert lines[1].split() == ["Old", "robots", "2026-08-01"] + assert lines[-1] == "3 by hand" + + def test_replacing_or_detaching_a_board_forgets_that_it_was_failing( store: Store, boards: Boards) -> None: from app.jobs.pull import FAILING, failing_boards From 28afc334c341700905441783092c4fab599ac673 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:02:19 +0200 Subject: [PATCH 03/11] Add, edit, check and delete a company by hand through the API, and show the by-hand fields in the list Co-Authored-By: Claude Sonnet 5.5 --- app/web/api.py | 122 ++++++++++++++++++++++++++++++----- tests/test_web.py | 161 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 259 insertions(+), 24 deletions(-) diff --git a/app/web/api.py b/app/web/api.py index 229f8b6..e27a3e2 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -10,6 +10,7 @@ This is roledesk's own API, over the same store. Every refusal is from __future__ import annotations from typing import Annotated, Any, Literal +from urllib.parse import urlsplit from fastapi import APIRouter, HTTPException, Query, Response from pydantic import BaseModel, Field @@ -17,7 +18,7 @@ from pydantic import BaseModel, Field from app import jobs from app.models import ( AFFINITY, Agent, Application, Company, Gate, Limits, Prompts, Role, RoleRow, - ScoredRole, Status, Subscription, Weights, company_key, + HandReason, ScoredRole, Status, Subscription, Weights, company_key, ) from app.storage import Store from app.web.deps import ConfigDep, StoreDep @@ -303,11 +304,17 @@ class CompanyOut(BaseModel): the board has failed on every fetch and why it last did, both empty while it answers. `from_search` marks a row that exists only because a search listed its roles and somebody judged it (`Store.set_affinity`): - no board, not even a detached one, and no `ats`, `careers_url` or - `domain`, which every way of following a company writes one of.""" + no board, not even a detached one, and no `ats`, `careers_url`, `domain` + or `hand_reason`, which every way of following a company or listing it by + hand writes one of (`jobs.sources.known_from_search_only`). `by_hand` + marks a company on the list checked by hand: no board shown and not + `from_search`. `hand_reason` says why (`app.models.HAND_REASONS`) and + `checked_at` when a person last looked at its careers page, "" for never.""" key: str name: str + domain: str = "" + notes: str = "" board: str subscription_id: int | None = None on: bool | None = None @@ -320,6 +327,9 @@ class CompanyOut(BaseModel): failing_since: str = "" failing_reason: str = "" from_search: bool = False + hand_reason: str = "" + checked_at: str = "" + by_hand: bool = False def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: @@ -337,29 +347,109 @@ def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: return boards -@router.get("/companies") -def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut]: - """Every company, with `affinity` as `agent` (the default agent when - absent) has it; 0 throughout when there is no agent.""" +def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]: boards = _company_boards(store) ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} counts = store.open_role_counts() failing = jobs.pull.failing_boards(store) out = [] - for c in store.companies(_agent_id(store, agent)): + for c in store.companies(agent): sub_id, host, on, target = boards.get(c.key, (None, "", None, "")) since, reason = failing.get(target, ("", "")) - out.append(CompanyOut(key=c.key, name=c.name, board=host, - subscription_id=sub_id, on=on, affinity=c.affinity, - open_roles=counts.get(c.key, 0), ats=c.ats, - ats_slug=c.ats_slug, careers_url=c.careers_url, - country=c.country, - failing_since=since, failing_reason=reason, - from_search=not (c.key in ever_boarded or c.ats - or c.careers_url or c.domain))) + from_search = jobs.sources.known_from_search_only(c, ever_boarded) + out.append(CompanyOut(key=c.key, name=c.name, domain=c.domain, notes=c.notes, + board=host, subscription_id=sub_id, on=on, + affinity=c.affinity, open_roles=counts.get(c.key, 0), + ats=c.ats, ats_slug=c.ats_slug, careers_url=c.careers_url, + country=c.country, failing_since=since, failing_reason=reason, + from_search=from_search, hand_reason=c.hand_reason, + checked_at=c.checked_at, + by_hand=c.key not in boards and not from_search)) return out +def _one_company(store: Store, key: str) -> CompanyOut: + """`key` as the list shows it, with the default agent's affinity.""" + return next(c for c in _company_outs(store, _agent_id(store, None)) if c.key == key) + + +@router.get("/companies") +def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut]: + """Every company, with `affinity` as `agent` (the default agent when + absent) has it; 0 throughout when there is no agent.""" + return _company_outs(store, _agent_id(store, agent)) + + +class NewCompanyBody(BaseModel): + name: str + careers_url: str = "" + notes: str = "" + hand_reason: HandReason = "added" + + +@router.post("/companies", status_code=201) +def add_company(body: NewCompanyBody, store: StoreDep) -> CompanyOut: + """A company put on the list checked by hand: no board, `hand_reason` + `added` unless it says another. 409 when its key, from the name, is taken.""" + try: + company = jobs.sources.add_by_hand(store, body.name, careers_url=body.careers_url, + notes=body.notes, hand_reason=body.hand_reason) + except jobs.sources.Conflict as exc: + raise HTTPException(409, str(exc)) from exc + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return _one_company(store, company.key) + + +class CompanyPatch(BaseModel): + name: str | None = None + careers_url: str | None = None + notes: str | None = None + hand_reason: HandReason | None = None + + +@router.patch("/companies/{key}") +def edit_company(key: str, body: CompanyPatch, store: StoreDep) -> CompanyOut: + """Change any of a company's name, careers link, notes and reason; a field + left out stays. The key never changes, and `name` only for a company + without a board: 409 for one with a subscription, shown or not (see + `jobs.sources.edit_company`).""" + fields = body.model_dump(exclude_unset=True) + if any(v is None for v in fields.values()): + raise HTTPException(422, "A field is left out, not null.") + try: + jobs.sources.edit_company(store, key, **fields) + except KeyError: + raise HTTPException(404, f"no such company: {key}") from None + except jobs.sources.Conflict as exc: + raise HTTPException(409, str(exc)) from exc + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return _one_company(store, key) + + +@router.post("/companies/{key}/checked") +def company_checked(key: str, store: StoreDep) -> CompanyOut: + """A person has just looked at the company's careers page.""" + if store.company(key) is None: + raise HTTPException(404, f"no such company: {key}") + store.update_company(key, checked_at=store.now()) + return _one_company(store, key) + + +@router.delete("/companies/{key}", status_code=204) +def remove_company(key: str, store: StoreDep) -> Response: + """Delete a company and every agent's stance on it. 409 for one with a + board shown or with open roles; detach the board first.""" + try: + jobs.sources.remove_company(store, key) + except KeyError: + raise HTTPException(404, f"no such company: {key}") from None + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + return Response(status_code=204) + + class CompanyBoardBody(BaseModel): board: str diff --git a/tests/test_web.py b/tests/test_web.py index 975b72a..2a6066f 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -421,24 +421,27 @@ def test_companies_lists_board_and_open_role_count( store.add_company("Acme", key="acme", ats="greenhouse", country="DK") store.upsert_role(posting("1", company="Acme"), sub) store.add_company("Other", key="other", domain="other.example") # no board found - store.add_company("Novo", key="novo", ats="successfactors", # followed, not readable + store.add_company("Novo", key="novo", ats="taleo", # followed, not readable careers_url="https://novo.example/careers") res = client.get("/api/companies") assert res.status_code == 200 [row] = [c for c in res.json() if c["key"] == "acme"] - assert row == {"key": "acme", "name": "Acme", "board": "greenhouse", - "subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1, - "ats": "greenhouse", "ats_slug": "", "careers_url": "", - "country": "DK", "failing_since": "", "failing_reason": "", - "from_search": False} + assert row == {"key": "acme", "name": "Acme", "domain": "", "notes": "", + "board": "greenhouse", "subscription_id": sub.id, "on": True, + "affinity": 0, "open_roles": 1, "ats": "greenhouse", "ats_slug": "", + "careers_url": "", "country": "DK", "failing_since": "", + "failing_reason": "", "from_search": False, "hand_reason": "", + "checked_at": "", "by_hand": False} [other] = [c for c in res.json() if c["key"] == "other"] assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( "", None, None, "") [novo] = [c for c in res.json() if c["key"] == "novo"] assert (novo["board"], novo["subscription_id"], novo["ats"], novo["careers_url"]) == ( - "", None, "successfactors", "https://novo.example/careers") + "", None, "taleo", "https://novo.example/careers") assert not any(c["from_search"] for c in res.json()) + # Companies with no board shown, unread or not, are the list checked by hand. + assert {c["key"] for c in res.json() if c["by_hand"]} == {"other", "novo"} def test_judging_a_company_known_only_from_a_search_creates_its_row( @@ -1214,10 +1217,152 @@ def test_a_companys_board_is_set_replaced_and_detached( assert client.post("/api/companies/nope/board", json={"board": "lever:acme"}).status_code == 404 assert client.delete("/api/companies/nope/board").status_code == 404 - bad = client.post("/api/companies/acme/board", json={"board": "workday:acme"}) + bad = client.post("/api/companies/acme/board", json={"board": "taleo:acme"}) assert bad.status_code == 422 and "host:slug" in bad.json()["detail"] +def test_a_company_is_added_by_hand_with_a_reason_and_a_link( + client: TestClient, store: Store) -> None: + res = client.post("/api/companies", json={ + "name": "Revolut Ltd", "careers_url": " https://revolut.example/careers ", + "notes": "wants a login"}) + assert res.status_code == 201, res.text + assert res.json() == { + "key": "revolut", "name": "Revolut Ltd", "domain": "", "notes": "wants a login", + "board": "", "subscription_id": None, "on": None, "affinity": 0, "open_roles": 0, + "ats": "", "ats_slug": "", "careers_url": "https://revolut.example/careers", + "country": "", "failing_since": "", "failing_reason": "", "from_search": False, + "hand_reason": "added", "checked_at": "", "by_hand": True} + assert store.company("revolut").hand_reason == "added" # type: ignore[union-attr] + + other = client.post("/api/companies", json={"name": "Meta", "hand_reason": "blocked"}) + assert other.status_code == 201 and other.json()["hand_reason"] == "blocked" + # The key comes from the name, so a legal suffix or case is the same company. + assert client.post("/api/companies", json={"name": "REVOLUT"}).status_code == 409 + assert store.company("revolut").notes == "wants a login" # type: ignore[union-attr] + + +def test_adding_a_company_by_hand_refuses_a_bad_link_reason_or_name( + client: TestClient, store: Store) -> None: + for link in ("javascript:alert(1)", "ftp://x.example/jobs", "revolut.example/careers", + "https://"): + res = client.post("/api/companies", json={"name": "Revolut", "careers_url": link}) + assert res.status_code == 422, link + assert client.post("/api/companies", json={"name": "Revolut", "careers_url": ""} + ).status_code == 201 + assert client.post("/api/companies", json={"name": "Meta", "hand_reason": "lazy"} + ).status_code == 422 + assert client.post("/api/companies", json={"name": " "}).status_code == 422 + assert client.post("/api/companies", json={}).status_code == 422 + assert [c.key for c in store.companies()] == ["revolut"] + + +def test_a_company_from_a_search_only_is_no_longer_one_once_it_has_a_reason( + client: TestClient, store: Store) -> None: + store.save_agent("A") + search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex") + item = posting("1", company="Novo") + item.source = "search" + store.upsert_role(item, search) + client.post("/api/companies/novo/affinity", json={"affinity": 1}) + [row] = client.get("/api/companies").json() + assert (row["from_search"], row["by_hand"]) == (True, False) + + res = client.patch("/api/companies/novo", json={"hand_reason": "unread"}) + assert res.status_code == 200 + assert (res.json()["from_search"], res.json()["by_hand"]) == (False, True) + + +def test_a_company_is_edited_and_only_the_fields_sent_change( + client: TestClient, store: Store) -> None: + store.add_company("Acme", key="acme", careers_url="https://acme.example/jobs", + notes="old", hand_reason="unread") + res = client.patch("/api/companies/acme", json={"notes": "new", "name": "Acme Labs"}) + assert res.status_code == 200, res.text + body = res.json() + assert (body["key"], body["name"], body["notes"], body["careers_url"], + body["hand_reason"]) == ("acme", "Acme Labs", "new", "https://acme.example/jobs", + "unread") + # A link and a reason can be cleared, since "" is a value. + res = client.patch("/api/companies/acme", json={"careers_url": "", "hand_reason": ""}) + assert (res.json()["careers_url"], res.json()["hand_reason"]) == ("", "") + assert client.patch("/api/companies/acme", json={}).status_code == 200 + + assert client.patch("/api/companies/acme", json={"careers_url": "javascript:x"} + ).status_code == 422 + assert client.patch("/api/companies/acme", json={"hand_reason": "lazy"}).status_code == 422 + assert client.patch("/api/companies/acme", json={"name": " "}).status_code == 422 + assert client.patch("/api/companies/acme", json={"notes": None}).status_code == 422 + assert client.patch("/api/companies/nope", json={"notes": "x"}).status_code == 404 + assert store.company("acme").name == "Acme Labs" # type: ignore[union-attr] + + +def test_a_company_with_a_board_keeps_its_name_and_takes_the_other_edits( + client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + import plugins.ats as ats + + monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) + store.add_company("Acme", key="acme") + assert client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"} + ).status_code == 200 + + res = client.patch("/api/companies/acme", json={"name": "Acme Labs"}) + assert res.status_code == 409 and "detach" in res.json()["detail"] + assert client.patch("/api/companies/acme", json={"name": "Acme"}).status_code == 200 + assert client.patch("/api/companies/acme", json={"notes": "hi"}).status_code == 200 + # Detached, the subscription is still there and the name stays fixed. + client.delete("/api/companies/acme/board") + assert client.patch("/api/companies/acme", json={"name": "Acme Labs"}).status_code == 409 + assert store.company("acme").name == "Acme" # type: ignore[union-attr] + + +def test_checking_a_company_stamps_now(client: TestClient, store: Store) -> None: + clock = Clock() + store.clock = clock + store.add_company("Acme", key="acme", hand_reason="unread") + res = client.post("/api/companies/acme/checked") + assert res.status_code == 200 + assert res.json()["checked_at"] == "2026-09-23T09:00:00+00:00" + clock.tick(48) + assert client.post("/api/companies/acme/checked").json()["checked_at"] == ( + "2026-09-25T09:00:00+00:00") + assert client.post("/api/companies/nope/checked").status_code == 404 + + +def test_a_company_is_deleted_unless_it_is_read( + client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + import plugins.ats as ats + + monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) + agent = store.save_agent("A") + store.add_company("Meta", key="meta", hand_reason="blocked") + store.set_affinity(agent.id, "meta", 1) + res = client.delete("/api/companies/meta") + assert res.status_code == 204 and res.content == b"" + assert store.company("meta") is None + assert store.company("meta", agent.id) is None + assert client.delete("/api/companies/meta").status_code == 404 + assert "meta" not in {c["key"] for c in client.get("/api/companies").json()} + + # A board shown: refused until it is detached; a detached board is no board. + store.add_company("Acme", key="acme") + sub_id = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"} + ).json()["source"]["id"] + res = client.delete("/api/companies/acme") + assert res.status_code == 409 and "greenhouse:acme" in res.json()["detail"] + store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id)) + client.delete("/api/companies/acme/board") + assert client.delete("/api/companies/acme").status_code == 204 + + # Open roles hold a company even where no board is on: they would be + # orphaned from the list. + store.add_company("Beta", key="beta") + store.upsert_role(posting("2", company="Beta")) + res = client.delete("/api/companies/beta") + assert res.status_code == 409 and "open roles" in res.json()["detail"] + assert store.company("beta") is not None + + def test_the_pull_a_source_switch_queues_is_refused_quietly_while_one_runs( app: FastAPI, store: Store) -> None: """The four actions that queue a `pull` (switching a source on, following From 0d11ce02782f7e22321eca4b423a3c7cd80ffa90 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 19:23:20 +0200 Subject: [PATCH 04/11] Record the Watchlist tab in the decisions Co-Authored-By: Claude Fable 5.1 --- docs/decisions.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/decisions.md b/docs/decisions.md index 0de8fa7..874d6a9 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -152,6 +152,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Two tabs and a cog.** `TABS` in `app/web/pages.py` becomes Roles and Applications; the other six pages (Agents, Companies, Sources, Jobs, Schedules, Messages, Account) move behind a settings cog at `/settings/`, because nine tabs wrapped to three rows on a phone and six of them were setup or system pages. +**Watchlist is a third tab: the companies the app cannot read, rated yes or no.** The owner goes through the careers pages of such companies by hand and wanted that list one click from Roles rather than behind the cog, so `TABS` has three entries and the phone bar's tabs are tighter. A company is on it when it has no board subscription and is not known from a search only, or when its board is failing: the rule of the Company boards filter "Not readable yet", read from `GET /api/companies`. The rating is `companies.watch` (migration 15: NULL unrated, 1 on the list, 0 not), global and apart from affinity, which is per agent and ranks roles the app does read. `careers_opened_at` is stamped when the careers link is followed, and On the list sorts by it, never opened first, so a round starts with what is overdue. A row rated under Unrated leaves the list at once. On a phone the search field of Roles is left a few letters wide by the third tab, so while it has the focus the tabs and the queue chip step aside. + **Settings are three sub-tabs: Search agents, Sources, Account.** Six sub-tabs were three setup pages and three system pages for two things, what the app ranks by and where roles come from, so each is one page now and Account stays its own. **The old paths' 301s are gone.** `MOVED` answered 301 for the twelve paths above, kept for the few days between the move and this cut, on the assumption that a bookmark should still land. A browser caches a 301 indefinitely, so a path a future page reused would be unreachable through no fault of that page; a one-person app with no outstanding bookmarks does not need to carry that risk. From 161e89c9557ec1ffb6f88971ddd422253be734da Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 19:25:46 +0200 Subject: [PATCH 05/11] Sort the by-hand list by the Watchlist's careers_opened_at and print the watch rating Co-Authored-By: Claude Sonnet 5.5 --- app/cli.py | 18 +++++++++++------- app/jobs/sources.py | 6 +++--- tests/test_sources.py | 24 ++++++++++++++---------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/app/cli.py b/app/cli.py index 35b3cbe..fe92c5d 100644 --- a/app/cli.py +++ b/app/cli.py @@ -409,12 +409,14 @@ def edit_company(store: Store, args: argparse.Namespace) -> int: def company_checked(store: Store, key: str) -> int: - """`app companies checked acme`: its careers page was looked at just now.""" + """`app companies checked acme`: its careers page was looked at just now, + which the Watchlist keeps as `careers_opened_at`, the same stamp opening + the page from there sets.""" if store.company(key) is None: print(f"companies checked: no company {key!r}", file=sys.stderr) return 1 - company = store.update_company(key, checked_at=store.now()) - print(json.dumps({"key": key, "checked_at": company.checked_at})) + company = store.update_company(key, careers_opened_at=store.now()) + print(json.dumps({"key": key, "careers_opened_at": company.careers_opened_at})) return 0 @@ -437,14 +439,16 @@ def remove_company(store: Store, key: str) -> int: def list_by_hand(store: Store) -> int: """`app companies by-hand`: the companies with no board shown that are not - known from a search only, one line each, never checked first and then the - longest ago, as the Sources page lists them; see `jobs.sources.by_hand`.""" + known from a search only, one line each, never opened first and then the + longest ago; `watch` is the Watchlist's rating, yes, no or a dash for + unrated. See `jobs.sources.by_hand`.""" from app import jobs rows = jobs.sources.by_hand(store) for c in rows: - checked = c.checked_at[:10] or "never" - print(f"{c.name:30} {c.hand_reason or '-':8} {checked:10} {c.careers_url}") + watch = "-" if c.watch is None else "yes" if c.watch else "no" + opened = c.careers_opened_at[:10] or "never" + print(f"{c.name:30} {c.hand_reason or '-':8} {watch:4} {opened:10} {c.careers_url}") print(f"{len(rows)} by hand") return 0 diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 9e566b6..76ceb5e 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -169,13 +169,13 @@ def known_from_search_only(company: Company, ever_boarded: set[str]) -> bool: def by_hand(store: Store) -> list[Company]: """The list checked by hand: companies with no board shown that are not - known from a search only. Never checked first, then the longest ago; the - name breaks a tie.""" + known from a search only. Never opened first, then the longest ago + (`careers_opened_at`, the Watchlist's stamp); the name breaks a tie.""" boards = store.company_boards() ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} rows = [c for c in store.companies() if c.key not in boards and not known_from_search_only(c, ever_boarded)] - return sorted(rows, key=lambda c: (c.checked_at, c.name.casefold())) + return sorted(rows, key=lambda c: (c.careers_opened_at, c.name.casefold())) class Conflict(ValueError): diff --git a/tests/test_sources.py b/tests/test_sources.py index dfedad0..d98bced 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -373,14 +373,14 @@ def test_remove_company_refuses_a_board_shown_and_open_roles( sources.remove_company(store, "open") -def test_by_hand_lists_the_unboarded_never_checked_first( +def test_by_hand_lists_the_unboarded_never_opened_first( store: Store, boards: Boards) -> None: sources.add_by_hand(store, "Zeta") sources.add_by_hand(store, "Beta") sources.add_by_hand(store, "Old") sources.add_by_hand(store, "Recent") - store.update_company("old", checked_at="2026-08-01T00:00:00+00:00") - store.update_company("recent", checked_at="2026-09-20T00:00:00+00:00") + store.update_company("old", careers_opened_at="2026-08-01T00:00:00+00:00") + store.update_company("recent", careers_opened_at="2026-09-20T00:00:00+00:00") sources.set_board(store, "read", "greenhouse:acme", name="Read") # a board: not listed store.add_company("Pleo", key="pleo", ats="teamtailor") # unread, no reason: listed store.add_company("Quiet", key="quiet") # from a search only @@ -486,8 +486,8 @@ def test_cli_edit_checked_and_remove_a_company_by_hand( assert "no company 'nobody'" in capsys.readouterr().err assert _cli(tmp_path, "checked", "acme") == 0 - stamp = json.loads(capsys.readouterr().out)["checked_at"] - assert stamp and store.company("acme").checked_at == stamp # type: ignore[union-attr] + stamp = json.loads(capsys.readouterr().out)["careers_opened_at"] + assert stamp and store.company("acme").careers_opened_at == stamp # type: ignore[union-attr] assert _cli(tmp_path, "checked", "nobody") == 1 capsys.readouterr() @@ -499,20 +499,24 @@ def test_cli_edit_checked_and_remove_a_company_by_hand( assert _cli(tmp_path, "remove", "acme") == 1 -def test_cli_by_hand_prints_never_checked_first_then_the_oldest_check( +def test_cli_by_hand_prints_never_opened_first_then_the_oldest_and_the_watch_rating( store: Store, boards: Boards, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: store.add_company("Recent", careers_url="https://recent.example/jobs", - hand_reason="gone", checked_at="2026-09-20T08:00:00+00:00") - store.add_company("Old", hand_reason="robots", checked_at="2026-08-01T08:00:00+00:00") + hand_reason="gone") + store.add_company("Old", hand_reason="robots") store.add_company("Never", careers_url="https://never.example/jobs", hand_reason="added") + store.update_company("recent", careers_opened_at="2026-09-20T08:00:00+00:00", watch=0) + store.update_company("old", careers_opened_at="2026-08-01T08:00:00+00:00", watch=1) sources.set_board(store, "read", "greenhouse:acme", name="Read") assert _cli(tmp_path, "by-hand") == 0 lines = capsys.readouterr().out.splitlines() assert [ln.split()[0] for ln in lines[:-1]] == ["Never", "Old", "Recent"] - assert lines[0].split() == ["Never", "added", "never", "https://never.example/jobs"] - assert lines[1].split() == ["Old", "robots", "2026-08-01"] + assert lines[0].split() == ["Never", "added", "-", "never", "https://never.example/jobs"] + assert lines[1].split() == ["Old", "robots", "yes", "2026-08-01"] + assert lines[2].split() == ["Recent", "gone", "no", "2026-09-20", + "https://recent.example/jobs"] assert lines[-1] == "3 by hand" From 1ebfab1b997d620ad241388c0147845efe63e422 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 19:26:08 +0200 Subject: [PATCH 06/11] Bring the by-hand plan in line with the Watchlist tab and migration 16 Co-Authored-By: Claude Sonnet 5.5 --- docs/plan-hosts-and-by-hand.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/plan-hosts-and-by-hand.md b/docs/plan-hosts-and-by-hand.md index e5c5cf3..fd478f6 100644 --- a/docs/plan-hosts-and-by-hand.md +++ b/docs/plan-hosts-and-by-hand.md @@ -38,7 +38,7 @@ The 49 were missed because the slug is not the company's name (`BoschGroup`, `ji **No reader is built on a key lifted from a site's script.** Maersk's API, New Work's and Welcome to the Jungle's search index answer only with a key that the page's JavaScript carries. They go on the by-hand list. -**The by-hand list is the `companies` table.** A company is on it when it has no shown board and is not known from a search only. `careers_url` and `notes` exist already; migration 15 adds `hand_reason` and `checked_at`. No second table, since a company has one careers link and the list needs no history of checks. +**The by-hand list is the `companies` table, and the Watchlist tab is the list.** A company is on it when it has no shown board and is not known from a search only. `careers_url`, `notes`, `watch` (the rating) and `careers_opened_at` exist already, the last being the date a person last looked at the careers page; migration 16 adds `hand_reason`. There is no `checked_at`, since opening the page from the Watchlist stamps the date already. No second table, since a company has one careers link and the list needs no history of checks. **A link typed by hand is kept.** `add_discovered` and the seed fill `careers_url` only when it is empty. @@ -75,16 +75,15 @@ def probe_ok(body: bytes) -> bool: ... Slugs: Workday `lego.wd103/LEGO_External`; SuccessFactors, Phenom and the sitemap reader take the site's hostname, with a path where the site has one (`jobs.teradyne.com/MiR`); Oracle `/`. -Company JSON, in `GET /api/companies`: `CompanyOut` gains `domain`, `notes`, `hand_reason`, `checked_at` and `by_hand` (bool). `hand_reason` is one of `''`, `unread` (readable, no parser yet), `robots`, `blocked` (the site answers a program with a challenge), `key`, `group`, `gone`, `added` (by hand or by an import, not looked at). `from_search` is false for a company with a `hand_reason`. +Company JSON, in `GET /api/companies`: `CompanyOut` has `domain`, `city`, `watch` and `careers_opened_at` from the Watchlist, and gains `notes`, `hand_reason` and `by_hand` (bool, no board shown and not `from_search`). `hand_reason` is one of `''`, `unread` (readable, no parser yet), `robots`, `blocked` (the site answers a program with a challenge), `key`, `group`, `gone`, `added` (by hand or by an import, not looked at). `from_search` is false for a company with a `hand_reason`. | Endpoint | Body | Answer | |---|---|---| | `POST /api/companies` | `name`, `careers_url`, `notes`, `hand_reason` (default `added`) | 201 `CompanyOut`; 409 when the key is taken | | `PATCH /api/companies/{key}` | any of `name`, `careers_url`, `notes`, `hand_reason` | `CompanyOut`; the key stays; 409 on `name` for a company with a board | -| `POST /api/companies/{key}/checked` | none | `CompanyOut` with `checked_at` now | | `DELETE /api/companies/{key}` | none | 204; 409 when it has a board or open roles; its `agent_companies` rows go with it | -`careers_url` is http or https, as `safeUrl` demands on the page. +`POST /api/companies/{key}/careers` and `POST /api/companies/{key}/opened` of the Watchlist stay and answer a `Company`; `POST .../careers` and `PATCH` check the link with the same function (`check_careers_url` in `app/jobs/sources.py`). `careers_url` is http or https, as `safeUrl` demands on the page. A person looking at the page is `POST .../opened`, which stamps `careers_opened_at`. ## Work packages @@ -94,7 +93,7 @@ Wave 0, in parallel, from today's `main`: |---|---|---|---| | 0 Host modules | `wp0-hosts` | Opus | `plugins/ats.py`, `plugins/atshosts/`, `app/jobs/discover.py`, `tests/test_ats.py`, `tests/test_discover.py`, `tests/test_atshosts.py` | | 1 By-hand backend | `wp1-byhand-api` | Sonnet | `app/models.py`, `app/storage/schema.py`, `app/storage/store.py`, `app/web/api.py`, `app/jobs/sources.py`, `app/cli.py`, `tests/test_storage.py`, `tests/test_web.py`, `tests/test_sources.py` | -| 2 By-hand page | `wp2-byhand-ui` | Opus | `app/templates/settings/sources.html`, `app/static/byhand.js`, `app/static/companies.js`, `app/static/app.css`, `app/web/pages.py` | +| 2b By-hand editing | `wp2b-byhand-ui` | Opus | `app/static/watchlist.js`, `app/templates/watchlist.html`, `app/static/app.css`, `tests/test_pages.py` | Wave 1, in parallel, from `main` after WP0 is merged. Each owns its modules in `plugins/atshosts/`, `tests/test_ats_.py` and `tests/fixtures/ats/*`, and nothing else: @@ -111,9 +110,9 @@ Wave 2 is decided after wave 1 by the roles each company would add: a module per **WP0.** The eight hosts move into modules with no change in behaviour: the existing tests pass with edits to imports only. `careers_url`, `is_known_target`, `_allowed_host` and `_paged` lose their Teamtailor branches to the module. `_get` takes `headers`. `detail` and the robots check are added to the shared code with tests against a fake host. `discover.py` takes `READABLE`, `HOSTS`, `_PROBES` and the body checks of `probe` from the modules; the hosts it names but cannot read stay in a table of its own. Discovery gains Lever's EU host. The tests that use `workday:` or `successfactors` as the unreadable host use `taleo` instead. -**WP1.** Migration 15, the four endpoints, `CompanyOut`'s new fields, `remove_company` taking the `agent_companies` rows, `careers_url` filled only when empty in `add_discovered` and `seed_companies`, the seed reading `hand_reason`, and `app companies add`, `edit`, `checked` and `remove` in the CLI. The `workday:` and `successfactors` fixtures in its three test files become `taleo`. +**WP1.** Migration 16 with `hand_reason` only, the endpoints `POST /api/companies`, `PATCH` and `DELETE`, `CompanyOut`'s new fields, `remove_company` taking the `agent_companies` rows, `careers_url` filled only when empty in `add_discovered` and `seed_companies`, the seed reading `hand_reason`, `check_careers_url` as the one rule for a typed link, and `app companies add`, `edit`, `checked`, `by-hand` and `remove` in the CLI, where `checked` stamps `careers_opened_at` and `by-hand` lists the never opened first. The `workday:` and `successfactors` fixtures in its three test files become `taleo`. -**WP2.** A section "Check by hand" in the Sources rail, after Company boards: a count, a New company button, and rows sorted by `checked_at`, never checked first. A row shows the name, the affinity control, the reason, the careers link, the note, "checked 3 days ago", and Checked, Edit and Remove. New and Edit use the form pattern of the job boards section (`sources.js`: a `