The ats boards only read companies someone listed, and the seed list is US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs saved queries on boards that span companies, pulled in the same task as the boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU, free key, built to its documented API). Jobnet and Jooble were checked and left out. `country` is now filled from the board's location line by a lookup, with EU for Europe-wide roles, so country gates work without extract. `app companies prune --in` unsubscribes boards with no open role in a region. The starter agents and searches are Copenhagen and Europe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
433 lines
19 KiB
Python
433 lines
19 KiB
Python
"""The command line.
|
|
|
|
app serve [--host H] [--port P] [--reload]
|
|
app job pull|extract|embed|score [--payload JSON]
|
|
app worker [--workers http=8,cpu=1] [--until-idle]
|
|
app companies seed|add|check|prune
|
|
app searches add|list|remove
|
|
app db init
|
|
|
|
`app job` queues the work on a queue of its own and drains it here, so a
|
|
cascade such as `pull` runs every stage it starts before the command returns,
|
|
and prints one line per stage. `app worker` is the boilerplate's: the same
|
|
queue with no web app, and the schedules fired from it.
|
|
|
|
`python -m app` is the same thing, for a checkout with no console script.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from app import config as config_mod
|
|
|
|
if TYPE_CHECKING:
|
|
from app.storage import Store
|
|
|
|
#: The one plugin this app ships lives outside the `app` package, in
|
|
#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a
|
|
#: user's own drop-in plugin would be. `companies` and `job pull` both want
|
|
#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as
|
|
#: well as the `Source` class, so the repo root goes on `sys.path` once, here,
|
|
#: rather than each caller guessing where `plugins/` is relative to `cwd`.
|
|
_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_ROOT))
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
|
|
parser.add_argument("-c", "--config", default=None,
|
|
help="the YAML file to read (default: $APP_CONFIG, then config.yaml)")
|
|
parser.add_argument("-v", "--verbose", action="store_true")
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
|
|
serve = sub.add_parser("serve", help="run the web app")
|
|
serve.add_argument("--host", default="", help="override config server.host")
|
|
serve.add_argument("--port", type=int, default=0, help="override config server.port")
|
|
serve.add_argument("--reload", action="store_true", help="restart on a source change")
|
|
|
|
job = sub.add_parser("job", help="queue a job, task or cascade and drain the queue")
|
|
job.add_argument("kind", help="pull, extract, embed, score, or any registered job")
|
|
job.add_argument("--payload", default="{}",
|
|
help="JSON spread over a job handler's keyword arguments")
|
|
|
|
worker = sub.add_parser("worker", help="drain the job table with no web app")
|
|
worker.add_argument("--workers", default="", help="resource=count,... e.g. cpu=2")
|
|
worker.add_argument("--until-idle", action="store_true",
|
|
help="run what is queued, then exit")
|
|
|
|
companies = sub.add_parser("companies", help="the target company list")
|
|
companies_sub = companies.add_subparsers(dest="action", required=True)
|
|
seed = companies_sub.add_parser(
|
|
"seed", help="import seed/companies.json, subscribing to each board")
|
|
seed.add_argument("--path", default="seed/companies.json")
|
|
add = companies_sub.add_parser("add", help="add one company from its board")
|
|
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
|
|
add.add_argument("name")
|
|
companies_sub.add_parser("check", help="report boards that do not answer")
|
|
prune = companies_sub.add_parser(
|
|
"prune", help="unsubscribe every board with no open role in the region")
|
|
prune.add_argument("--in", dest="region", default="europe",
|
|
help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)")
|
|
prune.add_argument("--apply", action="store_true",
|
|
help="unsubscribe and close their roles; without it, only list them")
|
|
|
|
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
|
searches_sub = searches.add_subparsers(dest="action", required=True)
|
|
search_add = searches_sub.add_parser(
|
|
"add", help="run a search once, then save it for every pull")
|
|
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
|
|
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
|
|
search_add.add_argument("--q", required=True,
|
|
help="the query, in the provider's own syntax")
|
|
search_add.add_argument("--area", type=int, default=0,
|
|
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
|
|
search_add.add_argument("--country", default="",
|
|
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
|
|
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
|
|
search_add.add_argument("--pages", type=int, default=0,
|
|
help="result pages per pull (default 5, at most 10)")
|
|
searches_sub.add_parser("list", help="every saved search and its open roles")
|
|
search_remove = searches_sub.add_parser(
|
|
"remove", help="stop running a search; its roles stay until they go stale")
|
|
search_remove.add_argument("name")
|
|
|
|
db = sub.add_parser("db", help="database housekeeping")
|
|
db.add_argument("action", choices=["init"])
|
|
return parser
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
try:
|
|
if args.command == "serve":
|
|
return serve(args)
|
|
if args.command == "job":
|
|
return run_job(args)
|
|
if args.command == "worker":
|
|
return run_worker(args)
|
|
if args.command == "companies":
|
|
return run_companies(args)
|
|
if args.command == "searches":
|
|
return run_searches(args)
|
|
return init_db(args)
|
|
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
|
print(exc, file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
def serve(args: argparse.Namespace) -> int:
|
|
import uvicorn
|
|
|
|
config = config_mod.load(args.config) # a config error here, not in uvicorn
|
|
host = args.host or config.server.host
|
|
port = args.port or config.server.port
|
|
level = str(config.log_level).lower()
|
|
if args.reload:
|
|
# The reloader builds the app in a fresh process from an import
|
|
# string, and that process reads the config from $APP_CONFIG.
|
|
if args.config:
|
|
os.environ["APP_CONFIG"] = str(args.config)
|
|
uvicorn.run("app.main:app", host=host, port=port, reload=True, log_level=level)
|
|
return 0
|
|
from app.main import create_app
|
|
|
|
uvicorn.run(create_app(config), host=host, port=port, log_level=level)
|
|
return 0
|
|
|
|
|
|
def run_job(args: argparse.Namespace) -> int:
|
|
"""Queue `kind` (a cascade such as `pull`, a task from `pipeline.TASKS`,
|
|
or a job), run everything queued until the queue is idle, and print how
|
|
it ended: one line per task, from each task's final job. `pull` is a
|
|
cascade, so it prints a line for every stage it went through.
|
|
|
|
The queue fires no schedules: the command asked for one piece of work."""
|
|
from app import jobs
|
|
from app.storage import Store
|
|
from pipeline import FINAL, JOBS, TASKS, JobQueue, Jobs
|
|
|
|
cascade = jobs.cascade
|
|
|
|
try:
|
|
payload: dict[str, Any] = json.loads(args.payload or "{}")
|
|
except json.JSONDecodeError as exc:
|
|
print(f"--payload is not JSON: {exc}", file=sys.stderr)
|
|
return 2
|
|
chained = args.kind in cascade.CASCADES
|
|
if not chained and args.kind not in TASKS and args.kind not in JOBS:
|
|
known = sorted(set(JOBS) | set(TASKS))
|
|
print(f"no job kind {args.kind!r}. Have: {', '.join(known)}", file=sys.stderr)
|
|
return 2
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
queue = JobQueue(Jobs(str(config.database)), config, store,
|
|
workers=config.workers.model_dump(),
|
|
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
|
|
tz=config.timezone, schedules=False)
|
|
if chained or args.kind not in JOBS:
|
|
task_id = queue.start_task(cascade.chain(args.kind) if chained
|
|
else TASKS[args.kind]())
|
|
queue.drain()
|
|
started = queue.jobs.of_task(task_id)
|
|
if not started:
|
|
print(json.dumps({"task": args.kind, "result": "nothing to do"}))
|
|
return 0
|
|
whole = queue.jobs.of_cascade(started[0].cascade_id)
|
|
finals = [j for j in whole if j.kind == FINAL]
|
|
for final in finals:
|
|
# A task's own jobs say what the work did; the final job only
|
|
# tallies how they ended. Up to five, so a pull of twenty board
|
|
# batches stays one readable line.
|
|
work = [j.result for j in whole if j.task_id == final.task_id
|
|
and j.kind != FINAL][:5]
|
|
print(json.dumps({"id": final.id, "state": final.state,
|
|
"result": final.result, "work": work,
|
|
"error": final.error}))
|
|
return 0 if all(f.state == "done" for f in finals) else 1
|
|
queued = queue.enqueue(args.kind, payload)
|
|
queue.drain()
|
|
if queued is not None:
|
|
done = queue.jobs.get(queued.id)
|
|
if done is not None:
|
|
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
|
|
"error": done.error}))
|
|
return 0 if done.state == "done" else 1
|
|
return 0
|
|
|
|
|
|
def run_worker(args: argparse.Namespace) -> int:
|
|
"""`python -m pipeline.worker` with this app's settings filled in: the
|
|
bootstrap, the pools from `workers:`, and the timezone, without which a
|
|
`daily at 06:00` schedule would run at 06:00 on the machine's clock."""
|
|
from app.jobs import BOOTSTRAP
|
|
from pipeline.worker import main as worker_main
|
|
|
|
config = config_mod.load(args.config)
|
|
argv = ["--db", str(config.database), "--bootstrap", BOOTSTRAP,
|
|
"--bootstrap-arg", str(args.config or config.path or ""),
|
|
"--timezone", str(config.timezone)]
|
|
argv += ["--workers", args.workers or
|
|
",".join(f"{name}={n}" for name, n in config.workers.model_dump().items())]
|
|
if args.until_idle:
|
|
argv.append("--until-idle")
|
|
return worker_main(argv)
|
|
|
|
|
|
def run_companies(args: argparse.Namespace) -> int:
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "seed":
|
|
print(json.dumps(seed_companies(store, args.path)))
|
|
return 0
|
|
if args.action == "add":
|
|
return add_company(store, args.board, args.name)
|
|
if args.action == "prune":
|
|
return prune_companies(store, args.region, apply=args.apply)
|
|
return check_companies(store)
|
|
|
|
|
|
def seed_companies(store: Store, path: str) -> dict[str, int]:
|
|
"""Import `seed/companies.json`: one `companies` row and one `ats`
|
|
subscription per entry, skipping a key that already exists. Not
|
|
idempotent on edits to an existing row — re-running it only ever adds
|
|
what is missing."""
|
|
from plugins.ats import is_known_target
|
|
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
added = skipped = 0
|
|
for row in data.get("companies", []):
|
|
key, name, board = row["key"], row["name"], row["board"]
|
|
if store.company(key) is not None:
|
|
skipped += 1
|
|
continue
|
|
if not is_known_target(board):
|
|
print(f"companies seed: {key}: {board!r} is not a known board; skipped",
|
|
file=sys.stderr)
|
|
skipped += 1
|
|
continue
|
|
store.add_company(name, key=key, affinity=int(row.get("affinity", 0)),
|
|
careers_url=row.get("careers_url", ""))
|
|
host, _, _slug = board.partition(":")
|
|
try:
|
|
store.add_subscription("ats", board, label=name, company=name, board=host)
|
|
except ValueError as exc:
|
|
# The company row landed; the subscription is the only thing
|
|
# missing, and worth saying rather than losing silently.
|
|
print(f"companies seed: {key}: {exc}", file=sys.stderr)
|
|
added += 1
|
|
return {"added": added, "skipped": skipped, "total": len(data.get("companies", []))}
|
|
|
|
|
|
def add_company(store: Store, board: str, name: str) -> int:
|
|
"""`app companies add greenhouse:acme "Acme"`: fetch the board first
|
|
and refuse if it does not answer, is not a known host, or the company
|
|
already exists. Port of seedgen's `-add`."""
|
|
from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target
|
|
from app.models import company_key
|
|
|
|
if not is_known_target(board):
|
|
print(f"companies add: {board!r} is not host:slug for a known ATS host",
|
|
file=sys.stderr)
|
|
return 1
|
|
key = company_key(name)
|
|
if store.company(key) is not None:
|
|
print(f"companies add: {key!r} already exists", file=sys.stderr)
|
|
return 1
|
|
try:
|
|
rows = fetch_board(board)
|
|
except Exception as exc:
|
|
print(f"companies add: {board} did not answer: {exc}", file=sys.stderr)
|
|
return 1
|
|
if not rows:
|
|
print(f"companies add: {board} answered but lists no roles; nothing added",
|
|
file=sys.stderr)
|
|
return 1
|
|
host, _, _slug = board.partition(":")
|
|
store.add_company(name, key=key, careers_url=careers_url_of(board))
|
|
try:
|
|
store.add_subscription("ats", board, label=name, company=name, board=host)
|
|
except ValueError as exc:
|
|
print(f"companies add: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": key, "board": board, "roles": len(rows)}))
|
|
return 0
|
|
|
|
|
|
def check_companies(store: Store) -> int:
|
|
"""`app companies check`: every `ats` board, fetched once. Prints a
|
|
line per board that did not answer and exits 1 if any did not."""
|
|
from plugins.ats import fetch_board
|
|
|
|
subs = store.subscriptions(source="ats")
|
|
dead = []
|
|
for sub in subs:
|
|
try:
|
|
fetch_board(sub.target)
|
|
except Exception as exc:
|
|
dead.append((sub.target, sub.label, str(exc)))
|
|
for target, label, reason in dead:
|
|
print(f"{target} ({label}): {reason}")
|
|
print(f"checked {len(subs)} boards, {len(dead)} dead")
|
|
return 1 if dead else 0
|
|
|
|
|
|
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
|
|
"""`app companies prune --in DK,EU`: every subscribed board none of whose
|
|
open roles is in the region, by `country`, which this fills from `place`
|
|
first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles,
|
|
since a board no longer pulled would otherwise keep them open for good.
|
|
`app companies seed` never brings one back: it skips a company that exists.
|
|
|
|
A board whose open roles all name no country is left alone: it says
|
|
nothing about where that company hires."""
|
|
from app.models import EUROPE
|
|
|
|
filled = store.backfill_country()
|
|
countries = store.open_countries_by_subscription()
|
|
wanted = {c.strip().upper() for c in region.split(",") if c.strip()}
|
|
if "EUROPE" in wanted:
|
|
wanted = (wanted - {"EUROPE"}) | set(EUROPE)
|
|
pruned = [(sub, sum(countries[sub.id].values()))
|
|
for sub in store.subscriptions(source="ats", fetchable=True)
|
|
if set(countries.get(sub.id, {})) - {""}
|
|
and not wanted.intersection(countries[sub.id])]
|
|
closed = 0
|
|
for sub, count in sorted(pruned, key=lambda p: -p[1]):
|
|
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
|
|
if apply:
|
|
store.unsubscribe(sub.id)
|
|
closed += store.close_subscription(sub.id)
|
|
kept = len(store.subscriptions(source="ats", fetchable=True))
|
|
print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply,
|
|
"roles_closed": closed, "boards_left": kept}))
|
|
return 0
|
|
|
|
|
|
def run_searches(args: argparse.Namespace) -> int:
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "add":
|
|
return add_search(config, store, args)
|
|
if args.action == "remove":
|
|
sub = store.subscription_for("search", args.name)
|
|
if sub is None:
|
|
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
|
|
return 1
|
|
store.unsubscribe(sub.id)
|
|
print(json.dumps({"removed": sub.label}))
|
|
return 0
|
|
countries = store.open_countries_by_subscription()
|
|
for sub in store.subscriptions(source="search"):
|
|
removed = "" if sub.subscribed and sub.fetch else " (removed)"
|
|
open_roles = sum(countries.get(sub.id, {}).values())
|
|
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
|
|
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
|
|
return 0
|
|
|
|
|
|
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
|
|
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
|
|
--area 1`: run the search once and refuse if it fails or finds nothing, so
|
|
a typo in a query is found now rather than by an empty tab later."""
|
|
from app.jobs import search_target
|
|
from plugins.search import run_search
|
|
|
|
params: dict[str, Any] = {"q": args.q}
|
|
if args.provider == "jobindex" and args.area:
|
|
params["area"] = args.area
|
|
if args.provider == "adzuna":
|
|
params["country"] = args.country.lower()
|
|
if args.where:
|
|
params["where"] = args.where
|
|
if args.pages:
|
|
params["pages"] = args.pages
|
|
target = search_target(args.provider, args.name)
|
|
try:
|
|
rows, complete = run_search(target, params, dict(config.sources.get("search") or {}))
|
|
except Exception as exc:
|
|
print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr)
|
|
return 1
|
|
if not rows:
|
|
print("searches add: the search found nothing; not saved", file=sys.stderr)
|
|
return 1
|
|
try:
|
|
store.add_subscription("search", target, label=args.name, board=args.provider,
|
|
params=params)
|
|
except ValueError as exc:
|
|
print(f"searches add: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"search": args.name, "target": target, "roles": len(rows),
|
|
"complete": complete}, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
def init_db(args: argparse.Namespace) -> int:
|
|
"""Create the file, or bring it to the current schema."""
|
|
from app.storage import LATEST, Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
applied = store.migrations_applied
|
|
print(f"{store.path}: schema v{LATEST}"
|
|
+ (f", applied {applied}" if applied else ""))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|