roledesk job extract reads open, unanswered roles 20 to a call and fills place, country, remote, seniority, salary, currency, function, permit and languages, only into columns the board left empty. Schema v2 adds roles.permit, roles.languages and roles.extract_tries. Off by default. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
222 lines
8.8 KiB
Python
222 lines
8.8 KiB
Python
"""`roledesk serve | job pull|extract | companies seed|add|check | db init`."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
from roledesk import config as config_mod
|
|
|
|
if TYPE_CHECKING:
|
|
from roledesk.storage import Store
|
|
|
|
#: The one plugin this app ships lives outside the `roledesk` package, in
|
|
#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a
|
|
#: user's own drop-in plugin would be. `companies` and `job pull` both want
|
|
#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as
|
|
#: well as the `Source` class, so the repo root goes on `sys.path` once, here,
|
|
#: rather than each caller guessing where `plugins/` is relative to `cwd`.
|
|
_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_ROOT))
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(prog="roledesk",
|
|
description="A job-search engine on desk.")
|
|
parser.add_argument("-c", "--config", default=None, help="path to roles.yaml")
|
|
parser.add_argument("-v", "--verbose", action="store_true")
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
|
|
serve = sub.add_parser("serve", help="run the web app")
|
|
serve.add_argument("--host", default="127.0.0.1")
|
|
serve.add_argument("--port", type=int, default=8790)
|
|
|
|
job = sub.add_parser("job", help="queue a job or task and drain the queue once")
|
|
job.add_argument("kind", choices=["pull", "extract"])
|
|
|
|
companies = sub.add_parser("companies", help="the target company list")
|
|
companies_sub = companies.add_subparsers(dest="action", required=True)
|
|
seed = companies_sub.add_parser(
|
|
"seed", help="import seed/companies.json, subscribing to each board")
|
|
seed.add_argument("--path", default="seed/companies.json")
|
|
add = companies_sub.add_parser("add", help="add one company from its board")
|
|
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
|
|
add.add_argument("name")
|
|
companies_sub.add_parser("check", help="report boards that do not answer")
|
|
|
|
db = sub.add_parser("db", help="database housekeeping")
|
|
db.add_argument("action", choices=["init"])
|
|
return parser
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
try:
|
|
if args.command == "serve":
|
|
from roledesk.web import serve
|
|
config_mod.load(args.config) # a config error here, not in uvicorn
|
|
return serve(args.host, args.port, args.config)
|
|
if args.command == "job":
|
|
return run_job(args)
|
|
if args.command == "companies":
|
|
return run_companies(args)
|
|
return init_db(args)
|
|
except FileNotFoundError as exc:
|
|
print(exc, file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
def run_job(args: argparse.Namespace) -> int:
|
|
"""Queue `kind` — a job, or a task from `pipeline.TASKS` such as `pull` —
|
|
then run everything queued until the queue is idle, and print how it
|
|
ended. A task's own result is its final job's, which is the one whose
|
|
`task_id` equals the run this started."""
|
|
from pipeline import FINAL, JobQueue, Jobs, TASKS
|
|
from roledesk import jobs
|
|
from roledesk.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
queue = JobQueue(Jobs(config.database), config, store,
|
|
workers=config.workers.model_dump(),
|
|
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
|
|
if args.kind in TASKS:
|
|
task_id = queue.start_task(TASKS[args.kind]())
|
|
queue.drain()
|
|
final = next((j for j in queue.jobs.of_task(task_id) if j.kind == FINAL), None)
|
|
if final is not None:
|
|
print(json.dumps({"id": final.id, "state": final.state,
|
|
"result": final.result, "error": final.error}))
|
|
return 0 if final.state == "done" else 1
|
|
return 0
|
|
queued = queue.enqueue(args.kind)
|
|
queue.drain()
|
|
if queued is not None:
|
|
done = queue.jobs.get(queued.id)
|
|
if done is not None:
|
|
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
|
|
"error": done.error}))
|
|
return 0 if done.state == "done" else 1
|
|
return 0
|
|
|
|
|
|
def run_companies(args: argparse.Namespace) -> int:
|
|
from roledesk.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "seed":
|
|
print(json.dumps(seed_companies(store, args.path)))
|
|
return 0
|
|
if args.action == "add":
|
|
return add_company(store, args.board, args.name)
|
|
return check_companies(store)
|
|
|
|
|
|
def seed_companies(store: Store, path: str) -> dict[str, int]:
|
|
"""Import `seed/companies.json`: one `companies` row and one `ats`
|
|
subscription per entry, skipping a key that already exists. Not
|
|
idempotent on edits to an existing row — re-running it only ever adds
|
|
what is missing."""
|
|
from plugins.ats import is_known_target
|
|
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
added = skipped = 0
|
|
for row in data.get("companies", []):
|
|
key, name, board = row["key"], row["name"], row["board"]
|
|
if store.company(key) is not None:
|
|
skipped += 1
|
|
continue
|
|
if not is_known_target(board):
|
|
print(f"companies seed: {key}: {board!r} is not a known board; skipped",
|
|
file=sys.stderr)
|
|
skipped += 1
|
|
continue
|
|
store.add_company(name, key=key, affinity=int(row.get("affinity", 0)),
|
|
careers_url=row.get("careers_url", ""))
|
|
host, _, _slug = board.partition(":")
|
|
try:
|
|
store.add_subscription("ats", board, label=name, company=name, board=host)
|
|
except ValueError as exc:
|
|
# The company row landed; the subscription is the only thing
|
|
# missing, and worth saying rather than losing silently.
|
|
print(f"companies seed: {key}: {exc}", file=sys.stderr)
|
|
added += 1
|
|
return {"added": added, "skipped": skipped, "total": len(data.get("companies", []))}
|
|
|
|
|
|
def add_company(store: Store, board: str, name: str) -> int:
|
|
"""`roledesk companies add greenhouse:acme "Acme"`: fetch the board first
|
|
and refuse if it does not answer, is not a known host, or the company
|
|
already exists. Port of seedgen's `-add`."""
|
|
from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target
|
|
from roledesk.models import company_key
|
|
|
|
if not is_known_target(board):
|
|
print(f"companies add: {board!r} is not host:slug for a known ATS host",
|
|
file=sys.stderr)
|
|
return 1
|
|
key = company_key(name)
|
|
if store.company(key) is not None:
|
|
print(f"companies add: {key!r} already exists", file=sys.stderr)
|
|
return 1
|
|
try:
|
|
rows = fetch_board(board)
|
|
except Exception as exc:
|
|
print(f"companies add: {board} did not answer: {exc}", file=sys.stderr)
|
|
return 1
|
|
if not rows:
|
|
print(f"companies add: {board} answered but lists no roles; nothing added",
|
|
file=sys.stderr)
|
|
return 1
|
|
host, _, _slug = board.partition(":")
|
|
store.add_company(name, key=key, careers_url=careers_url_of(board))
|
|
try:
|
|
store.add_subscription("ats", board, label=name, company=name, board=host)
|
|
except ValueError as exc:
|
|
print(f"companies add: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": key, "board": board, "roles": len(rows)}))
|
|
return 0
|
|
|
|
|
|
def check_companies(store: Store) -> int:
|
|
"""`roledesk companies check`: every `ats` board, fetched once. Prints a
|
|
line per board that did not answer and exits 1 if any did not."""
|
|
from plugins.ats import fetch_board
|
|
|
|
subs = store.subscriptions(source="ats")
|
|
dead = []
|
|
for sub in subs:
|
|
try:
|
|
fetch_board(sub.target)
|
|
except Exception as exc:
|
|
dead.append((sub.target, sub.label, str(exc)))
|
|
for target, label, reason in dead:
|
|
print(f"{target} ({label}): {reason}")
|
|
print(f"checked {len(subs)} boards, {len(dead)} dead")
|
|
return 1 if dead else 0
|
|
|
|
|
|
def init_db(args: argparse.Namespace) -> int:
|
|
"""Create the file, or bring it to the current schema."""
|
|
from roledesk.storage import LATEST, Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
applied = store.migrations_applied
|
|
print(f"{store.path}: schema v{LATEST}"
|
|
+ (f", applied {applied}" if applied else ""))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|