A second pull (schedule, Run button, `app job pull` in another process) could start while one was fetching and overwrite the stored run marker, so the first run's boards failed their stamp check or a settle closed on the other run's id lists. `claim_run` sets `pull:run` in one conditional statement, only while it equals `pull:settled` or is older than an hour; a refused `pull` job queues no stage and says so in its result and a notice, and the CLI exits 1. Batches carry the marker in their payload, the plugins stamp with it, and the settle reads it off its own batches' payloads before releasing the run. Co-Authored-By: Claude Opus <noreply@anthropic.com>
418 lines
18 KiB
Python
418 lines
18 KiB
Python
"""The command line.
|
|
|
|
app serve [--host H] [--port P] [--reload]
|
|
app job pull|extract|embed|score [--payload JSON]
|
|
app worker [--workers http=8,cpu=1] [--until-idle]
|
|
app companies seed|add|discover|check
|
|
app searches add|list|remove
|
|
app db init
|
|
|
|
`app job` queues the work on a queue of its own and drains it here, so a
|
|
cascade such as `pull` runs every stage it starts before the command returns,
|
|
and prints one line per stage. `app worker` is the boilerplate's: the same
|
|
queue with no web app, and the schedules fired from it.
|
|
|
|
`python -m app` is the same thing, for a checkout with no console script.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from app import config as config_mod
|
|
|
|
if TYPE_CHECKING:
|
|
from app.storage import Store
|
|
|
|
#: The one plugin this app ships lives outside the `app` package, in
|
|
#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a
|
|
#: user's own drop-in plugin would be. `companies` and `job pull` both want
|
|
#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as
|
|
#: well as the `Source` class, so the repo root goes on `sys.path` once, here,
|
|
#: rather than each caller guessing where `plugins/` is relative to `cwd`.
|
|
_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_ROOT))
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
|
|
parser.add_argument("-c", "--config", default=None,
|
|
help="the YAML file to read (default: $APP_CONFIG, then config.yaml)")
|
|
parser.add_argument("-v", "--verbose", action="store_true")
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
|
|
serve = sub.add_parser("serve", help="run the web app")
|
|
serve.add_argument("--host", default="", help="override config server.host")
|
|
serve.add_argument("--port", type=int, default=0, help="override config server.port")
|
|
serve.add_argument("--reload", action="store_true", help="restart on a source change")
|
|
|
|
job = sub.add_parser("job", help="queue a job, task or cascade and drain the queue")
|
|
job.add_argument("kind", help="pull, extract, embed, score, or any registered job")
|
|
job.add_argument("--payload", default="{}",
|
|
help="JSON spread over a job handler's keyword arguments")
|
|
|
|
worker = sub.add_parser("worker", help="drain the job table with no web app")
|
|
worker.add_argument("--workers", default="", help="resource=count,... e.g. cpu=2")
|
|
worker.add_argument("--until-idle", action="store_true",
|
|
help="run what is queued, then exit")
|
|
|
|
companies = sub.add_parser("companies", help="the target company list")
|
|
companies_sub = companies.add_subparsers(dest="action", required=True)
|
|
seed = companies_sub.add_parser(
|
|
"seed", help="import seed/companies.json, subscribing to each readable board")
|
|
seed.add_argument("--path", default="seed/companies.json")
|
|
add = companies_sub.add_parser("add", help="add one company from its board")
|
|
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
|
|
add.add_argument("name")
|
|
discover = companies_sub.add_parser(
|
|
"discover", help="find a company's board from its website; --add follows it")
|
|
discover.add_argument("name")
|
|
discover.add_argument("domain", help="its website, e.g. pleo.io")
|
|
discover.add_argument("--add", action="store_true",
|
|
help="add the company, with its board when one is readable")
|
|
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
|
|
|
|
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
|
searches_sub = searches.add_subparsers(dest="action", required=True)
|
|
search_add = searches_sub.add_parser(
|
|
"add", help="run a search once, then save it for every pull (or update it)")
|
|
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
|
|
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
|
|
search_add.add_argument("--q", required=True,
|
|
help="the query, in the provider's own syntax")
|
|
search_add.add_argument("--area", type=int, default=0,
|
|
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
|
|
search_add.add_argument("--country", default="",
|
|
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
|
|
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
|
|
search_add.add_argument("--pages", type=int, default=0,
|
|
help="result pages per pull (default 5, at most 10)")
|
|
searches_sub.add_parser("list", help="every saved search and its open roles")
|
|
search_remove = searches_sub.add_parser(
|
|
"remove", help="stop running a search and close its open roles")
|
|
search_remove.add_argument("name")
|
|
|
|
db = sub.add_parser("db", help="database housekeeping")
|
|
db.add_argument("action", choices=["init"])
|
|
return parser
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
try:
|
|
if args.command == "serve":
|
|
return serve(args)
|
|
if args.command == "job":
|
|
return run_job(args)
|
|
if args.command == "worker":
|
|
return run_worker(args)
|
|
if args.command == "companies":
|
|
return run_companies(args)
|
|
if args.command == "searches":
|
|
return run_searches(args)
|
|
return init_db(args)
|
|
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
|
print(exc, file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
def serve(args: argparse.Namespace) -> int:
|
|
import uvicorn
|
|
|
|
config = config_mod.load(args.config) # a config error here, not in uvicorn
|
|
host = args.host or config.server.host
|
|
port = args.port or config.server.port
|
|
level = str(config.log_level).lower()
|
|
if args.reload:
|
|
# The reloader builds the app in a fresh process from an import
|
|
# string, and that process reads the config from $APP_CONFIG.
|
|
if args.config:
|
|
os.environ["APP_CONFIG"] = str(args.config)
|
|
uvicorn.run("app.main:app", host=host, port=port, reload=True, log_level=level)
|
|
return 0
|
|
from app.main import create_app
|
|
|
|
uvicorn.run(create_app(config), host=host, port=port, log_level=level)
|
|
return 0
|
|
|
|
|
|
def run_job(args: argparse.Namespace) -> int:
|
|
"""Queue `kind` (a cascade such as `pull`, a task from `pipeline.TASKS`,
|
|
or a job), run everything queued until the queue is idle, and print how
|
|
it ended: one line per task, from each task's final job. `pull` is a
|
|
cascade, so it prints a line for every stage it went through.
|
|
|
|
The queue fires no schedules: the command asked for one piece of work."""
|
|
from app import jobs
|
|
from app.storage import Store
|
|
from pipeline import FINAL, JOBS, TASKS, JobQueue, Jobs
|
|
|
|
cascade = jobs.cascade
|
|
|
|
try:
|
|
payload: dict[str, Any] = json.loads(args.payload or "{}")
|
|
except json.JSONDecodeError as exc:
|
|
print(f"--payload is not JSON: {exc}", file=sys.stderr)
|
|
return 2
|
|
chained = args.kind in cascade.CASCADES
|
|
if not chained and args.kind not in TASKS and args.kind not in JOBS:
|
|
known = sorted(set(JOBS) | set(TASKS))
|
|
print(f"no job kind {args.kind!r}. Have: {', '.join(known)}", file=sys.stderr)
|
|
return 2
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
queue = JobQueue(Jobs(str(config.database)), config, store,
|
|
workers=config.workers.model_dump(),
|
|
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
|
|
tz=config.timezone, schedules=False)
|
|
if chained or args.kind not in JOBS:
|
|
first = cascade.chain(args.kind) if chained else TASKS[args.kind]()
|
|
if args.kind == "pull":
|
|
# The same claim the `pull` job makes, so a pull here and one in
|
|
# the server's worker cannot both fetch.
|
|
run = jobs.pull.claim_run(store)
|
|
if not run:
|
|
print(json.dumps({"task": "pull", "result": jobs.pull.ALREADY_RUNNING,
|
|
"running": jobs.pull.running(store)}))
|
|
return 1
|
|
first = cascade.pull_chain(run)
|
|
task_id = queue.start_task(first)
|
|
queue.drain()
|
|
started = queue.jobs.of_task(task_id)
|
|
if not started:
|
|
print(json.dumps({"task": args.kind, "result": "nothing to do"}))
|
|
return 0
|
|
whole = queue.jobs.of_cascade(started[0].cascade_id)
|
|
finals = [j for j in whole if j.kind == FINAL]
|
|
for final in finals:
|
|
# A task's own jobs say what the work did; the final job only
|
|
# tallies how they ended. Up to five, so a pull of twenty board
|
|
# batches stays one readable line.
|
|
work = [j.result for j in whole if j.task_id == final.task_id
|
|
and j.kind != FINAL][:5]
|
|
print(json.dumps({"id": final.id, "state": final.state,
|
|
"result": final.result, "work": work,
|
|
"error": final.error}))
|
|
return 0 if all(f.state == "done" for f in finals) else 1
|
|
queued = queue.enqueue(args.kind, payload)
|
|
queue.drain()
|
|
if queued is not None:
|
|
done = queue.jobs.get(queued.id)
|
|
if done is not None:
|
|
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
|
|
"error": done.error}))
|
|
return 0 if done.state == "done" else 1
|
|
return 0
|
|
|
|
|
|
def run_worker(args: argparse.Namespace) -> int:
|
|
"""`python -m pipeline.worker` with this app's settings filled in: the
|
|
bootstrap, the pools from `workers:`, and the timezone, without which a
|
|
`daily at 06:00` schedule would run at 06:00 on the machine's clock."""
|
|
from app.jobs import BOOTSTRAP
|
|
from pipeline.worker import main as worker_main
|
|
|
|
config = config_mod.load(args.config)
|
|
argv = ["--db", str(config.database), "--bootstrap", BOOTSTRAP,
|
|
"--bootstrap-arg", str(args.config or config.path or ""),
|
|
"--timezone", str(config.timezone)]
|
|
argv += ["--workers", args.workers or
|
|
",".join(f"{name}={n}" for name, n in config.workers.model_dump().items())]
|
|
if args.until_idle:
|
|
argv.append("--until-idle")
|
|
return worker_main(argv)
|
|
|
|
|
|
def run_companies(args: argparse.Namespace) -> int:
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "seed":
|
|
print(json.dumps(seed_companies(store, args.path)))
|
|
return 0
|
|
if args.action == "add":
|
|
return add_company(store, args.board, args.name)
|
|
if args.action == "discover":
|
|
return discover_company(store, args.name, args.domain, add=args.add)
|
|
return check_companies(store)
|
|
|
|
|
|
def seed_companies(store: Store, path: str) -> dict[str, int]:
|
|
"""Import `seed/companies.json`: one `companies` row per entry, and one
|
|
`ats` subscription for each entry with a readable `board`. An entry
|
|
without one is the point of the list as much as one with: a company worth
|
|
following whose careers site the app cannot read yet keeps its `ats`
|
|
host and `careers_url`, and the Companies section says so. A key that
|
|
already exists is skipped, so re-running only ever adds what is missing;
|
|
it never edits a row."""
|
|
from app.models import company_key
|
|
from plugins.ats import careers_url, is_known_target
|
|
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
added = readable = skipped = 0
|
|
for row in data.get("companies", []):
|
|
name = row["name"]
|
|
key = row.get("key") or company_key(name)
|
|
if store.company(key) is not None:
|
|
skipped += 1
|
|
continue
|
|
board = row.get("board") or ""
|
|
if board and not is_known_target(board):
|
|
print(f"companies seed: {key}: {board!r} is not a known board; kept without it",
|
|
file=sys.stderr)
|
|
board = ""
|
|
host = board.partition(":")[0]
|
|
store.add_company(
|
|
name, key=key, domain=row.get("domain", ""),
|
|
careers_url=row.get("careers_url") or (careers_url(board) if board else ""),
|
|
ats=row.get("ats") or host, country=row.get("country", ""),
|
|
city=row.get("city", ""))
|
|
added += 1
|
|
if not board:
|
|
continue
|
|
try:
|
|
store.add_subscription("ats", board, label=name, company=name, board=host)
|
|
readable += 1
|
|
except ValueError as exc:
|
|
# The company row landed; the subscription is the only thing
|
|
# missing, and worth saying rather than losing silently.
|
|
print(f"companies seed: {key}: {exc}", file=sys.stderr)
|
|
return {"added": added, "readable": readable, "skipped": skipped,
|
|
"total": len(data.get("companies", []))}
|
|
|
|
|
|
def discover_company(store: Store, name: str, domain: str, *, add: bool) -> int:
|
|
"""`app companies discover "Pleo" pleo.io [--add]`: what the company's
|
|
site says its board is, as JSON. With `--add`, the company is added the
|
|
way the seed adds it: with its board when one is readable, as a row the
|
|
app cannot read yet otherwise."""
|
|
from app import jobs
|
|
|
|
found = jobs.discover.discover(name, domain)
|
|
out: dict[str, Any] = {"name": name, "domain": domain, "board": found.board,
|
|
"ats": found.ats, "careers_url": found.careers_url,
|
|
"evidence": found.evidence}
|
|
if add:
|
|
try:
|
|
out["key"], out["roles"] = jobs.sources.add_discovered(store, name, domain, found)
|
|
except ValueError as exc:
|
|
print(f"companies discover: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps(out))
|
|
return 0
|
|
|
|
|
|
def add_company(store: Store, board: str, name: str) -> int:
|
|
"""`app companies add greenhouse:acme "Acme"`: fetch the board first
|
|
and refuse if it does not answer, is not a known host, or the company
|
|
already exists. Port of seedgen's `-add`."""
|
|
from app import jobs
|
|
from app.models import company_key
|
|
|
|
try:
|
|
sub, found = jobs.sources.add_board(store, board, name)
|
|
except ValueError as exc:
|
|
print(f"companies add: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": company_key(sub.company), "board": sub.target, "roles": found}))
|
|
return 0
|
|
|
|
|
|
def check_companies(store: Store) -> int:
|
|
"""`app companies check`: every `ats` board, fetched once. Prints a
|
|
line per board that did not answer now, answered incompletely, or has
|
|
been failing in the pull, with what the pull recorded beside the live
|
|
result, and exits 1 if any did not answer now."""
|
|
from app import jobs
|
|
from plugins.ats import failure_reason, fetch_board
|
|
|
|
subs = store.subscriptions(source="ats")
|
|
failing = jobs.pull.failing_boards(store)
|
|
dead = 0
|
|
for sub in subs:
|
|
try:
|
|
rows, complete = fetch_board(sub.target)
|
|
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
|
|
except Exception as exc:
|
|
complete, live = False, f"not answering: {failure_reason(exc)}"
|
|
dead += 1
|
|
since, reason = failing.get(sub.target, ("", ""))
|
|
if complete and not since:
|
|
continue
|
|
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
|
|
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
|
|
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
|
|
return 1 if dead else 0
|
|
|
|
|
|
def run_searches(args: argparse.Namespace) -> int:
|
|
from app import jobs
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "add":
|
|
return add_search(config, store, args)
|
|
if args.action == "remove":
|
|
sub = store.subscription_for("search", args.name)
|
|
if sub is None:
|
|
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
|
|
return 1
|
|
_, closed = jobs.sources.remove_search(store, sub.id)
|
|
print(json.dumps({"removed": sub.label, "roles_closed": closed}))
|
|
return 0
|
|
countries = store.open_countries_by_subscription()
|
|
for sub in store.subscriptions(source="search"):
|
|
removed = "" if sub.subscribed and sub.fetch else " (off)"
|
|
open_roles = sum(countries.get(sub.id, {}).values())
|
|
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
|
|
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
|
|
return 0
|
|
|
|
|
|
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
|
|
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
|
|
--area 1`: run the search once and save it only if it finds something.
|
|
A search of the same name is updated in place."""
|
|
from app import jobs
|
|
|
|
try:
|
|
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
|
|
country=args.country, where=args.where,
|
|
pages=args.pages)
|
|
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
|
|
params)
|
|
except ValueError as exc:
|
|
print(f"searches add: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"search": sub.label, "target": sub.target, "roles": found,
|
|
"complete": complete}, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
def init_db(args: argparse.Namespace) -> int:
|
|
"""Create the file, or bring it to the current schema."""
|
|
from app.storage import LATEST, Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
applied = store.migrations_applied
|
|
print(f"{store.path}: schema v{LATEST}"
|
|
+ (f", applied {applied}" if applied else ""))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|