diff --git a/README.md b/README.md index 7e55246..b921f40 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,12 @@ uv run app searches list uv run app companies discover "Pleo" pleo.io --add # find its board on its own site, then follow it uv run app companies set-board lunar greenhouse:lunar # attach a board to a company, or replace the one it has uv run app companies detach lunar # stop reading its board; the company stays +uv run app companies add "Revolut" --careers-url https://www.revolut.com/careers --reason blocked # one word: a company to read by hand +uv run app companies add greenhouse:lunar Lunar # two: a board and its company, which follows the company +uv run app companies by-hand # the companies read by hand, never opened first, with the Watchlist's rating +uv run app companies edit revolut --note "needs a login" # name, link, note or reason +uv run app companies checked revolut # its careers page was looked at just now; the Watchlist shows the date +uv run app companies remove revolut # refused while a board is on or it has open roles ``` The seed is a hand-picked list of European tech employers, not a list of boards: a company whose careers site runs on something the app cannot read yet (Workday, SuccessFactors, its own pages) is still a row, with that host and its slug there where one was found, shown under Companies as "not readable yet" with a link to its careers page, so the gap is visible and one parser at a time closes it. A row where nothing was found reads "board not found"; the "Not readable yet" filter shows both. Each row has a Set board control (Change once it has one) to attach, replace or detach its board by hand. Replacing or detaching closes the old board's open roles, and asks first when there are open roles to lose; a board is read once before it is kept, and one that lists no roles yet is kept too. Setting a board queues a pull right after, so its roles are read by the one already running or the very next one, rather than waiting for the schedule. `companies discover` reads a company's site for a link into one of the boards it knows, probes the hosts with a cheap per-slug endpoint only when no page names any host, and with `--add` sets the board of a new or existing company, or records the host it found when nothing readable was. A probe hit is a guess by name: check the roles it lists before trusting it for a common name. diff --git a/app/cli.py b/app/cli.py index 6017af6..fe92c5d 100644 --- a/app/cli.py +++ b/app/cli.py @@ -3,7 +3,7 @@ app serve [--host H] [--port P] [--reload] app job pull|extract|embed|score [--payload JSON] app worker [--workers http=8,cpu=1] [--until-idle] - app companies seed|add|set-board|detach|discover|check + app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check app searches add|list|remove app agents list|prompt app stages [extract|summarise on|off] @@ -44,6 +44,8 @@ if str(_ROOT) not in sys.path: def build_parser() -> argparse.ArgumentParser: + from app.models import HAND_REASONS + parser = argparse.ArgumentParser( prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.") parser.add_argument("-c", "--config", default=None, @@ -71,10 +73,34 @@ def build_parser() -> argparse.ArgumentParser: seed = companies_sub.add_parser( "seed", help="import seed/companies.json, and the boards it names for rows without one") seed.add_argument("--path", default="seed/companies.json") + # One word after `add` is a company for the list checked by hand, two are + # a board and its company, which is what `add` meant before that list. add = companies_sub.add_parser( - "add", help="follow a company by its board, attaching or replacing an existing one's") - add.add_argument("board", help="host:slug, e.g. greenhouse:acme") - add.add_argument("name") + "add", help="add NAME to the list checked by hand, or with BOARD NAME follow a " + "company by its board, attaching or replacing an existing one's") + add.add_argument("first", metavar="NAME|BOARD", + help="the company, or host:slug, e.g. greenhouse:acme, before its name") + add.add_argument("second", metavar="NAME", nargs="?") + add.add_argument("--careers-url", default="", help="its careers page, http or https") + add.add_argument("--note", default="") + add.add_argument("--reason", default="added", choices=HAND_REASONS, metavar="REASON", + help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}") + edit = companies_sub.add_parser("edit", help="change a company's name, link, note or reason") + edit.add_argument("key") + edit.add_argument("--name") + edit.add_argument("--careers-url") + edit.add_argument("--note") + edit.add_argument("--reason", choices=HAND_REASONS, metavar="REASON", + help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}") + checked = companies_sub.add_parser( + "checked", help="record that its careers page was looked at just now") + checked.add_argument("key") + remove = companies_sub.add_parser( + "remove", help="delete a company with no board on and no open roles") + remove.add_argument("key") + companies_sub.add_parser( + "by-hand", help="the companies checked by hand, never checked first, then the " + "longest ago") set_board = companies_sub.add_parser( "set-board", help="attach a board to a company, or replace the one it has") set_board.add_argument("key", help="the company key, e.g. trade-republic") @@ -274,7 +300,15 @@ def run_companies(args: argparse.Namespace) -> int: if args.action == "seed": return seed_companies(store, args.path) if args.action == "add": - return add_company(store, args.board, args.name) + return add_company(store, args) + if args.action == "edit": + return edit_company(store, args) + if args.action == "checked": + return company_checked(store, args.key) + if args.action == "remove": + return remove_company(store, args.key) + if args.action == "by-hand": + return list_by_hand(store) if args.action == "set-board": return set_board(store, args.key, args.board) if args.action == "detach": @@ -318,7 +352,25 @@ def discover_company(store: Store, name: str, domain: str, *, add: bool) -> int: return 0 -def add_company(store: Store, board: str, name: str) -> int: +def add_company(store: Store, args: argparse.Namespace) -> int: + """`app companies add "Acme" [--careers-url URL] [--note TEXT] [--reason + REASON]` puts a company with no board on the list checked by hand; + `app companies add greenhouse:acme "Acme"` follows it by its board.""" + if args.second is not None: + return add_company_by_board(store, args.first, args.second) + from app import jobs + + try: + company = jobs.sources.add_by_hand(store, args.first, careers_url=args.careers_url, + notes=args.note, hand_reason=args.reason) + except ValueError as exc: + print(f"companies add: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": company.key, "hand_reason": company.hand_reason})) + return 0 + + +def add_company_by_board(store: Store, board: str, name: str) -> int: """`app companies add greenhouse:acme "Acme"`: fetch the board first and refuse if it does not answer or is not a known host. A company that exists gets the board attached or replaced. Port of seedgen's `-add`.""" @@ -334,6 +386,73 @@ def add_company(store: Store, board: str, name: str) -> int: return 0 +def edit_company(store: Store, args: argparse.Namespace) -> int: + """`app companies edit acme --careers-url https://acme.example/jobs`: only + the options given change; see `jobs.sources.edit_company`.""" + from app import jobs + + given = {"name": args.name, "careers_url": args.careers_url, "notes": args.note, + "hand_reason": args.reason} + try: + company = jobs.sources.edit_company( + store, args.key, **{f: v for f, v in given.items() if v is not None}) + except KeyError: + print(f"companies edit: no company {args.key!r}", file=sys.stderr) + return 1 + except ValueError as exc: + print(f"companies edit: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": company.key, "name": company.name, + "careers_url": company.careers_url, "notes": company.notes, + "hand_reason": company.hand_reason})) + return 0 + + +def company_checked(store: Store, key: str) -> int: + """`app companies checked acme`: its careers page was looked at just now, + which the Watchlist keeps as `careers_opened_at`, the same stamp opening + the page from there sets.""" + if store.company(key) is None: + print(f"companies checked: no company {key!r}", file=sys.stderr) + return 1 + company = store.update_company(key, careers_opened_at=store.now()) + print(json.dumps({"key": key, "careers_opened_at": company.careers_opened_at})) + return 0 + + +def remove_company(store: Store, key: str) -> int: + """`app companies remove acme`: refused for a company still read, see + `jobs.sources.remove_company`.""" + from app import jobs + + try: + jobs.sources.remove_company(store, key) + except KeyError: + print(f"companies remove: no company {key!r}", file=sys.stderr) + return 1 + except ValueError as exc: + print(f"companies remove: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": key, "removed": True})) + return 0 + + +def list_by_hand(store: Store) -> int: + """`app companies by-hand`: the companies with no board shown that are not + known from a search only, one line each, never opened first and then the + longest ago; `watch` is the Watchlist's rating, yes, no or a dash for + unrated. See `jobs.sources.by_hand`.""" + from app import jobs + + rows = jobs.sources.by_hand(store) + for c in rows: + watch = "-" if c.watch is None else "yes" if c.watch else "no" + opened = c.careers_opened_at[:10] or "never" + print(f"{c.name:30} {c.hand_reason or '-':8} {watch:4} {opened:10} {c.careers_url}") + print(f"{len(rows)} by hand") + return 0 + + def set_board(store: Store, key: str, board: str) -> int: """`app companies set-board trade-republic greenhouse:traderepublic`: attach the board to an existing company or replace the one it has.""" diff --git a/app/jobs/sources.py b/app/jobs/sources.py index f9bb354..cb73677 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -15,9 +15,10 @@ import json import os from pathlib import Path from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit from app.config import Config -from app.models import Subscription, company_key +from app.models import HAND_REASONS, Company, Subscription, company_key from app.storage import Store from app.jobs.pull import FAILING, search_target @@ -45,8 +46,9 @@ def set_board(store: Store, key: str, board: str, *, name: str = "", a company between openings is still one to follow. `check=False` skips the fetch, for the seed, whose boards were read once when it was written. - The row's `ats` and `ats_slug` become the board's host and slug, and its - `careers_url` the board's page where it was empty or the old board's. + The row's `ats` and `ats_slug` become the board's host and slug, its + `careers_url` the board's page where it was empty or the old board's, and + its `hand_reason` empty, since a company that is read is not read by hand. `fields` (domain, country, city, ...) fill only columns that are empty, so nothing a person or the seed wrote is overwritten. Row and subscription change in one transaction (`Store.set_company_board`), and replacing a @@ -81,7 +83,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "", writes["careers_url"] = careers_url(board) else: writes.setdefault("careers_url", careers_url(board)) - writes.update(ats=host, ats_slug=slug) + writes.update(ats=host, ats_slug=slug, hand_reason="") before = store.company_subscription(key) _company, sub, closed = store.set_company_board(key, board, host=host, name=name, fields=writes) @@ -154,12 +156,116 @@ def add_discovered(store: Store, name: str, domain: str, if sub is not None and sub.show: raise ValueError(f"{name} is read from {sub.target} and discovery found no " "readable board; nothing changed") + # A link somebody typed is kept: discovery's is only a guess at the page. store.update_company(key, ats=found.ats, ats_slug=found.ats_slug, - careers_url=found.careers_url or company.careers_url, + careers_url=company.careers_url or found.careers_url, **({"domain": domain} if not company.domain else {})) return key, 0 +def known_from_search_only(company: Company, ever_boarded: set[str]) -> bool: + """A row that exists only because a search listed its roles and somebody + judged it (`Store.set_affinity`): no board, not even a detached one + (`ever_boarded`, the keys of every `ats` subscription), and none of the + facts every way of following a company or listing it by hand writes.""" + return not (company.key in ever_boarded or company.ats or company.careers_url + or company.domain or company.hand_reason) + + +def by_hand(store: Store) -> list[Company]: + """The list checked by hand: companies with no board shown that are not + known from a search only. Never opened first, then the longest ago + (`careers_opened_at`, the Watchlist's stamp); the name breaks a tie.""" + boards = store.company_boards() + ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} + rows = [c for c in store.companies() + if c.key not in boards and not known_from_search_only(c, ever_boarded)] + return sorted(rows, key=lambda c: (c.careers_opened_at, c.name.casefold())) + + +class Conflict(ValueError): + """A change refused because of what the company already is, not because of + what was asked for: the pages answer it with 409, not 422.""" + + +def check_careers_url(url: str) -> str: + """`url` stripped; ValueError unless it is empty or an http(s) link. The + page puts it in an `href`, where `javascript:` would run.""" + url = url.strip() + if url: + parts = urlsplit(url) + if parts.scheme not in ("http", "https") or not parts.netloc: + raise ValueError("The careers link must start with http:// or https://.") + return url + + +def add_by_hand(store: Store, name: str, *, careers_url: str = "", notes: str = "", + hand_reason: str = "added") -> Company: + """A company with no board, put on the list checked by hand. Raises + Conflict when the key its name folds to is taken, ValueError for a name + with no letters, a link that is no http(s) URL or a reason outside + `HAND_REASONS`.""" + link = check_careers_url(careers_url) + key = company_key(name) + if key and store.company(key) is not None: + raise Conflict(f"{name.strip()} is on the list already as {key}.") + return store.add_company(name, key=key, careers_url=link, notes=notes, + hand_reason=hand_reason) + + +def edit_company(store: Store, key: str, **fields: str) -> Company: + """Change a company's `name`, `careers_url`, `notes` or `hand_reason`; the + key never changes. A new name is Conflict for a company with a board + subscription, shown or not, since the subscription is found by the key + its company name folds to and would be lost. Raises KeyError for an + unknown company and ValueError for a blank name, a link that is no http(s) + URL or a reason outside `HAND_REASONS`.""" + unknown = set(fields) - {"name", "careers_url", "notes", "hand_reason"} + if unknown: + raise ValueError(f"not a field to edit: {', '.join(sorted(unknown))}") + company = store.company(key) + if company is None: + raise KeyError(key) + if "careers_url" in fields: + fields["careers_url"] = check_careers_url(fields["careers_url"]) + if "name" in fields: + fields["name"] = fields["name"].strip() + if not fields["name"]: + raise ValueError("A company needs a name.") + if fields["name"] != company.name and store.company_subscription(key) is not None: + raise Conflict(f"{company.name} has a board, which is found by its name; " + "detach the board first.") + return store.update_company(key, **fields) + + +def remove_company(store: Store, key: str) -> None: + """Delete company `key` and every agent's stance on it. Raises KeyError + for an unknown company and Conflict for one still read: a company with + a board on (`show`, as `detach_board` decides) or with open roles, which + would otherwise vanish from the list with nothing said. A detached board + is no board, since nothing reads it and its roles are closed; its + subscription row stays, and follows the company if it comes back.""" + if store.company(key) is None: + raise KeyError(key) + sub = store.company_subscription(key) + if sub is not None and sub.show: + raise Conflict(f"{key} is read from {sub.target}; detach the board first") + if store.open_role_counts().get(key): + raise Conflict(f"{key} has open roles; detach its board first") + store.remove_company(key) + + +def _fill_empty(store: Store, key: str, **facts: str) -> None: + """Write each of `facts` to company `key` where the stored value is + empty: what a seed may add to a row that exists, since a person's link or + reason is never overwritten.""" + company = store.company(key) + assert company is not None + fill = {f: v for f, v in facts.items() if v and not getattr(company, f)} + if fill: + store.update_company(key, **fill) + + def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: """Import `seed/companies.json`: one `companies` row per entry, and its `board` through `set_board` where the entry names one. An entry without @@ -171,8 +277,10 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: existing row that never had a board gets the one the seed now names, so a seed update that gains boards reaches a database made before it. A row's board, once it has one (detached counts), and its affinity, notes and - status are never touched. The seed's boards are not fetched again: they - were read once when the seed was written, and `app companies check` + status are never touched; its `careers_url` and `hand_reason` are filled + where empty, so the seed can hand a row its link and its reason but never + overwrite the one a person set. The seed's boards are not fetched again: + they were read once when the seed was written, and `app companies check` finds the dead ones. Returns counts, and `problems`, one line per entry that could not land whole.""" from plugins.ats import is_known_target @@ -190,8 +298,15 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: board = "" facts = {f: str(row.get(f) or "") for f in ("domain", "country", "city", "careers_url", "ats", "ats_slug")} + reason = str(row.get("hand_reason") or "") + if reason not in HAND_REASONS: + problems.append(f"{key}: {reason!r} is not a hand_reason; kept without it") + reason = "" exists = store.company(key) is not None + if exists: + _fill_empty(store, key, hand_reason=reason) if exists and (not board or store.company_subscription(key) is not None): + _fill_empty(store, key, careers_url=facts["careers_url"]) skipped += 1 continue if board: @@ -204,13 +319,14 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: except ValueError as exc: problems.append(f"{key}: {exc}") if exists: + _fill_empty(store, key, careers_url=facts["careers_url"]) skipped += 1 continue # No board, or one that clashed: the company is still a row, and a # clashing board's host and slug are what discovery found. if board: facts["ats"], _, facts["ats_slug"] = board.partition(":") - store.add_company(name, key=key, **facts) + store.add_company(name, key=key, hand_reason=reason, **facts) added += 1 return {"added": added, "boards": boards, "skipped": skipped, "total": len(entries), "problems": problems} diff --git a/app/models.py b/app/models.py index 3378751..1e83ccd 100644 --- a/app/models.py +++ b/app/models.py @@ -644,6 +644,16 @@ STATUSES: tuple[str, ...] = get_args(Status) #: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score). AFFINITY = range(-1, 3) +#: Why a company is on the list checked by hand. `unread` is a host the app +#: could read and has no parser for yet, `robots` a site whose robots.txt +#: disallows it, `blocked` one that answers a program with a challenge, `key` +#: one whose API answers only with a key lifted from its own script, `group` a +#: company that sits inside its owner's board, `gone` one whose site is dead, +#: and `added` one put there by hand or by an import and not looked at. '' +#: is none of these: the company is read, or is not a matter for the list. +HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"] +HAND_REASONS: tuple[str, ...] = get_args(HandReason) + # -------------------------------------------------------------- posting dates @@ -929,6 +939,9 @@ class Company(Model): watch: int | None = None #: When the careers page was last opened from the Watchlist; "" never. careers_opened_at: str = "" + #: Why the company is read by hand, one of `HAND_REASONS`; "" when it is + #: not on that list. + hand_reason: str = "" created_at: str updated_at: str diff --git a/app/static/app.css b/app/static/app.css index 4766423..ead45d3 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -1130,7 +1130,8 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di .agent-form h2, .agent-form h3 { font-size: 1rem; margin: 0; } .agent-form label { display: flex; flex-direction: column; gap: .2rem; font-size: .9rem; } .agent-form label.inline { flex-direction: row; align-items: center; gap: .4rem; } -.agent-form input[type="text"], .agent-form textarea, .agent-form input[type="number"] { +.agent-form input[type="text"], .agent-form textarea, .agent-form input[type="number"], +.agent-form input[type="url"] { background: var(--bg); border: 1px solid var(--line); border-radius: .3rem; padding: .35rem .5rem; width: 100%; } @@ -1253,6 +1254,9 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di } .crow .board-edit input { flex: 1 1 8rem; min-width: 0; } .crow .sub .failing { color: var(--warn); } +/* A Watchlist row's note, a line of its own: the owner's line breaks kept, + and a pasted URL broken rather than widening a phone's page. */ +.crow .sub.notes { white-space: pre-line; overflow-wrap: anywhere; } .linky { padding: 0; border: 0; background: none; font-size: .8rem; diff --git a/app/static/watchlist.js b/app/static/watchlist.js index 92a896e..6416ca4 100644 --- a/app/static/watchlist.js +++ b/app/static/watchlist.js @@ -2,13 +2,20 @@ * going through their careers pages by hand. * * The list is app.js's shared company list (`/api/companies`), narrowed to - * the rule the "Not readable yet" filter of Company boards uses (no board - * subscription, not from a search), plus every company whose board is - * failing. The rating (`watch`: null unrated, 1 yes, 0 no) is global, not a - * search agent's affinity, and is set with `POST .../watch`. Opening the - * careers page stamps `careers_opened_at` (`POST .../opened`), so the row can - * say when, and the On the list filter sorts what was opened longest ago - * first. The filter is remembered per browser, and opens on Unrated. + * the by-hand list (`by_hand`: no board shown, not known from a search only), + * plus every company whose board is failing. The rating (`watch`: null + * unrated, 1 yes, 0 no) is global, not a search agent's affinity, and is set + * with `POST .../watch`. Opening the careers page stamps `careers_opened_at` + * (`POST .../opened`), so the row can say when, and the On the list filter + * sorts what was opened longest ago first. The filter is remembered per + * browser, and opens on Unrated. + * + * New company and Edit are the Job boards form of Sources (sources.js): Edit + * turns a row into the form from the page's `