"""The command line. app serve [--host H] [--port P] [--reload] app job pull|extract|embed|score [--payload JSON] app worker [--workers http=8,cpu=1] [--until-idle] app companies seed|add|check|prune app searches add|list|remove app db init `app job` queues the work on a queue of its own and drains it here, so a cascade such as `pull` runs every stage it starts before the command returns, and prints one line per stage. `app worker` is the boilerplate's: the same queue with no web app, and the schedules fired from it. `python -m app` is the same thing, for a checkout with no console script. """ from __future__ import annotations import argparse import json import logging import os import sys from pathlib import Path from typing import TYPE_CHECKING, Any from app import config as config_mod if TYPE_CHECKING: from app.storage import Store #: The one plugin this app ships lives outside the `app` package, in #: `plugins/`, so it is discovered by `desk.sources.registry` the same way a #: user's own drop-in plugin would be. `companies` and `job pull` both want #: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as #: well as the `Source` class, so the repo root goes on `sys.path` once, here, #: rather than each caller guessing where `plugins/` is relative to `cwd`. _ROOT = Path(__file__).resolve().parents[1] if str(_ROOT) not in sys.path: sys.path.insert(0, str(_ROOT)) def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.") parser.add_argument("-c", "--config", default=None, help="the YAML file to read (default: $APP_CONFIG, then config.yaml)") parser.add_argument("-v", "--verbose", action="store_true") sub = parser.add_subparsers(dest="command", required=True) serve = sub.add_parser("serve", help="run the web app") serve.add_argument("--host", default="", help="override config server.host") serve.add_argument("--port", type=int, default=0, help="override config server.port") serve.add_argument("--reload", action="store_true", help="restart on a source change") job = sub.add_parser("job", help="queue a job, task or cascade and drain the queue") job.add_argument("kind", help="pull, extract, embed, score, or any registered job") job.add_argument("--payload", default="{}", help="JSON spread over a job handler's keyword arguments") worker = sub.add_parser("worker", help="drain the job table with no web app") worker.add_argument("--workers", default="", help="resource=count,... e.g. cpu=2") worker.add_argument("--until-idle", action="store_true", help="run what is queued, then exit") companies = sub.add_parser("companies", help="the target company list") companies_sub = companies.add_subparsers(dest="action", required=True) seed = companies_sub.add_parser( "seed", help="import seed/companies.json, subscribing to each board") seed.add_argument("--path", default="seed/companies.json") add = companies_sub.add_parser("add", help="add one company from its board") add.add_argument("board", help="host:slug, e.g. greenhouse:acme") add.add_argument("name") companies_sub.add_parser("check", help="report boards that do not answer") prune = companies_sub.add_parser( "prune", help="unsubscribe every board with no open role in the region") prune.add_argument("--in", dest="region", default="europe", help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)") prune.add_argument("--apply", action="store_true", help="unsubscribe and close their roles; without it, only list them") searches = sub.add_parser("searches", help="saved searches on cross-company job boards") searches_sub = searches.add_subparsers(dest="action", required=True) search_add = searches_sub.add_parser( "add", help="run a search once, then save it for every pull") search_add.add_argument("provider", choices=["jobindex", "adzuna"]) search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"') search_add.add_argument("--q", required=True, help="the query, in the provider's own syntax") search_add.add_argument("--area", type=int, default=0, help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus") search_add.add_argument("--country", default="", help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch") search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin") search_add.add_argument("--pages", type=int, default=0, help="result pages per pull (default 5, at most 10)") searches_sub.add_parser("list", help="every saved search and its open roles") search_remove = searches_sub.add_parser( "remove", help="stop running a search; its roles stay until they go stale") search_remove.add_argument("name") db = sub.add_parser("db", help="database housekeeping") db.add_argument("action", choices=["init"]) return parser def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") try: if args.command == "serve": return serve(args) if args.command == "job": return run_job(args) if args.command == "worker": return run_worker(args) if args.command == "companies": return run_companies(args) if args.command == "searches": return run_searches(args) return init_db(args) except (FileNotFoundError, config_mod.ConfigError) as exc: print(exc, file=sys.stderr) return 1 def serve(args: argparse.Namespace) -> int: import uvicorn config = config_mod.load(args.config) # a config error here, not in uvicorn host = args.host or config.server.host port = args.port or config.server.port level = str(config.log_level).lower() if args.reload: # The reloader builds the app in a fresh process from an import # string, and that process reads the config from $APP_CONFIG. if args.config: os.environ["APP_CONFIG"] = str(args.config) uvicorn.run("app.main:app", host=host, port=port, reload=True, log_level=level) return 0 from app.main import create_app uvicorn.run(create_app(config), host=host, port=port, log_level=level) return 0 def run_job(args: argparse.Namespace) -> int: """Queue `kind` (a cascade such as `pull`, a task from `pipeline.TASKS`, or a job), run everything queued until the queue is idle, and print how it ended: one line per task, from each task's final job. `pull` is a cascade, so it prints a line for every stage it went through. The queue fires no schedules: the command asked for one piece of work.""" from app import jobs from app.storage import Store from pipeline import FINAL, JOBS, TASKS, JobQueue, Jobs cascade = jobs.cascade try: payload: dict[str, Any] = json.loads(args.payload or "{}") except json.JSONDecodeError as exc: print(f"--payload is not JSON: {exc}", file=sys.stderr) return 2 chained = args.kind in cascade.CASCADES if not chained and args.kind not in TASKS and args.kind not in JOBS: known = sorted(set(JOBS) | set(TASKS)) print(f"no job kind {args.kind!r}. Have: {', '.join(known)}", file=sys.stderr) return 2 config = config_mod.load(args.config) store = Store(config.database) queue = JobQueue(Jobs(str(config.database)), config, store, workers=config.workers.model_dump(), bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""), tz=config.timezone, schedules=False) if chained or args.kind not in JOBS: task_id = queue.start_task(cascade.chain(args.kind) if chained else TASKS[args.kind]()) queue.drain() started = queue.jobs.of_task(task_id) if not started: print(json.dumps({"task": args.kind, "result": "nothing to do"})) return 0 whole = queue.jobs.of_cascade(started[0].cascade_id) finals = [j for j in whole if j.kind == FINAL] for final in finals: # A task's own jobs say what the work did; the final job only # tallies how they ended. Up to five, so a pull of twenty board # batches stays one readable line. work = [j.result for j in whole if j.task_id == final.task_id and j.kind != FINAL][:5] print(json.dumps({"id": final.id, "state": final.state, "result": final.result, "work": work, "error": final.error})) return 0 if all(f.state == "done" for f in finals) else 1 queued = queue.enqueue(args.kind, payload) queue.drain() if queued is not None: done = queue.jobs.get(queued.id) if done is not None: print(json.dumps({"id": done.id, "state": done.state, "result": done.result, "error": done.error})) return 0 if done.state == "done" else 1 return 0 def run_worker(args: argparse.Namespace) -> int: """`python -m pipeline.worker` with this app's settings filled in: the bootstrap, the pools from `workers:`, and the timezone, without which a `daily at 06:00` schedule would run at 06:00 on the machine's clock.""" from app.jobs import BOOTSTRAP from pipeline.worker import main as worker_main config = config_mod.load(args.config) argv = ["--db", str(config.database), "--bootstrap", BOOTSTRAP, "--bootstrap-arg", str(args.config or config.path or ""), "--timezone", str(config.timezone)] argv += ["--workers", args.workers or ",".join(f"{name}={n}" for name, n in config.workers.model_dump().items())] if args.until_idle: argv.append("--until-idle") return worker_main(argv) def run_companies(args: argparse.Namespace) -> int: from app.storage import Store config = config_mod.load(args.config) store = Store(config.database) if args.action == "seed": print(json.dumps(seed_companies(store, args.path))) return 0 if args.action == "add": return add_company(store, args.board, args.name) if args.action == "prune": return prune_companies(store, args.region, apply=args.apply) return check_companies(store) def seed_companies(store: Store, path: str) -> dict[str, int]: """Import `seed/companies.json`: one `companies` row and one `ats` subscription per entry, skipping a key that already exists. Not idempotent on edits to an existing row — re-running it only ever adds what is missing.""" from plugins.ats import is_known_target data = json.loads(Path(path).read_text(encoding="utf-8")) added = skipped = 0 for row in data.get("companies", []): key, name, board = row["key"], row["name"], row["board"] if store.company(key) is not None: skipped += 1 continue if not is_known_target(board): print(f"companies seed: {key}: {board!r} is not a known board; skipped", file=sys.stderr) skipped += 1 continue store.add_company(name, key=key, affinity=int(row.get("affinity", 0)), careers_url=row.get("careers_url", "")) host, _, _slug = board.partition(":") try: store.add_subscription("ats", board, label=name, company=name, board=host) except ValueError as exc: # The company row landed; the subscription is the only thing # missing, and worth saying rather than losing silently. print(f"companies seed: {key}: {exc}", file=sys.stderr) added += 1 return {"added": added, "skipped": skipped, "total": len(data.get("companies", []))} def add_company(store: Store, board: str, name: str) -> int: """`app companies add greenhouse:acme "Acme"`: fetch the board first and refuse if it does not answer, is not a known host, or the company already exists. Port of seedgen's `-add`.""" from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target from app.models import company_key if not is_known_target(board): print(f"companies add: {board!r} is not host:slug for a known ATS host", file=sys.stderr) return 1 key = company_key(name) if store.company(key) is not None: print(f"companies add: {key!r} already exists", file=sys.stderr) return 1 try: rows = fetch_board(board) except Exception as exc: print(f"companies add: {board} did not answer: {exc}", file=sys.stderr) return 1 if not rows: print(f"companies add: {board} answered but lists no roles; nothing added", file=sys.stderr) return 1 host, _, _slug = board.partition(":") store.add_company(name, key=key, careers_url=careers_url_of(board)) try: store.add_subscription("ats", board, label=name, company=name, board=host) except ValueError as exc: print(f"companies add: {exc}", file=sys.stderr) return 1 print(json.dumps({"key": key, "board": board, "roles": len(rows)})) return 0 def check_companies(store: Store) -> int: """`app companies check`: every `ats` board, fetched once. Prints a line per board that did not answer and exits 1 if any did not.""" from plugins.ats import fetch_board subs = store.subscriptions(source="ats") dead = [] for sub in subs: try: fetch_board(sub.target) except Exception as exc: dead.append((sub.target, sub.label, str(exc))) for target, label, reason in dead: print(f"{target} ({label}): {reason}") print(f"checked {len(subs)} boards, {len(dead)} dead") return 1 if dead else 0 def prune_companies(store: Store, region: str, *, apply: bool) -> int: """`app companies prune --in DK,EU`: every subscribed board none of whose open roles is in the region, by `country`, which this fills from `place` first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles, since a board no longer pulled would otherwise keep them open for good. `app companies seed` never brings one back: it skips a company that exists. A board whose open roles all name no country is left alone: it says nothing about where that company hires.""" from app.models import EUROPE filled = store.backfill_country() countries = store.open_countries_by_subscription() wanted = {c.strip().upper() for c in region.split(",") if c.strip()} if "EUROPE" in wanted: wanted = (wanted - {"EUROPE"}) | set(EUROPE) pruned = [(sub, sum(countries[sub.id].values())) for sub in store.subscriptions(source="ats", fetchable=True) if set(countries.get(sub.id, {})) - {""} and not wanted.intersection(countries[sub.id])] closed = 0 for sub, count in sorted(pruned, key=lambda p: -p[1]): print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}") if apply: store.unsubscribe(sub.id) closed += store.close_subscription(sub.id) kept = len(store.subscriptions(source="ats", fetchable=True)) print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply, "roles_closed": closed, "boards_left": kept})) return 0 def run_searches(args: argparse.Namespace) -> int: from app.storage import Store config = config_mod.load(args.config) store = Store(config.database) if args.action == "add": return add_search(config, store, args) if args.action == "remove": sub = store.subscription_for("search", args.name) if sub is None: print(f"searches remove: no search named {args.name!r}", file=sys.stderr) return 1 store.unsubscribe(sub.id) print(json.dumps({"removed": sub.label})) return 0 countries = store.open_countries_by_subscription() for sub in store.subscriptions(source="search"): removed = "" if sub.subscribed and sub.fetch else " (removed)" open_roles = sum(countries.get(sub.id, {}).values()) print(f"{sub.label:30} {sub.target:40} {open_roles:5} open " f"{json.dumps(sub.params, ensure_ascii=False)}{removed}") return 0 def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int: """`app searches add jobindex "Copenhagen backend" --q '"backend developer"' --area 1`: run the search once and refuse if it fails or finds nothing, so a typo in a query is found now rather than by an empty tab later.""" from app.jobs import search_target from plugins.search import run_search params: dict[str, Any] = {"q": args.q} if args.provider == "jobindex" and args.area: params["area"] = args.area if args.provider == "adzuna": params["country"] = args.country.lower() if args.where: params["where"] = args.where if args.pages: params["pages"] = args.pages target = search_target(args.provider, args.name) try: rows, complete = run_search(target, params, dict(config.sources.get("search") or {})) except Exception as exc: print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr) return 1 if not rows: print("searches add: the search found nothing; not saved", file=sys.stderr) return 1 try: store.add_subscription("search", target, label=args.name, board=args.provider, params=params) except ValueError as exc: print(f"searches add: {exc}", file=sys.stderr) return 1 print(json.dumps({"search": args.name, "target": target, "roles": len(rows), "complete": complete}, ensure_ascii=False)) return 0 def init_db(args: argparse.Namespace) -> int: """Create the file, or bring it to the current schema.""" from app.storage import LATEST, Store config = config_mod.load(args.config) store = Store(config.database) applied = store.migrations_applied print(f"{store.path}: schema v{LATEST}" + (f", applied {applied}" if applied else "")) return 0 if __name__ == "__main__": raise SystemExit(main())