Files
jobboard/app/cli.py
T
WolfandClaude Fable 5.1 51d1e67ff9 A European seed of 362 employers, readable or not; discover finds a company's board from its site
The seed was jobhunt-hq's 186 US boards, chosen because they could be read. It is now a hand-picked list of European tech employers, with `board` filled by the new `app.jobs.discover` from each company's own careers site and every board fetched once before it landed: 175 have one, and the rest keep the ATS host their site runs on (Teamtailor 32, Workday 12, SuccessFactors 9, ...) and a careers URL, as rows the app cannot read yet. Companies shows them as "not readable yet" with a link, behind a filter, so the next parser to write is the host that filter shows most.

`app companies discover NAME DOMAIN [--add]` reads the home page, its careers links and the usual paths for a link into a known host, one level down on a careers page, and probes Greenhouse, Lever, Ashby, SmartRecruiters, Recruitee and Personio by slug when the pages link nothing. It is what the LinkedIn flow will call.

Migration 7 adds companies.ats, country and city. The Workable parser now reads the v3 shape its list endpoint actually sends (location object, department list, shortcode and no URL or text); the first populated board showed it.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-28 15:27:14 +02:00

430 lines
19 KiB
Python

"""The command line.
app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle]
app companies seed|add|discover|check|prune
app searches add|list|remove
app db init
`app job` queues the work on a queue of its own and drains it here, so a
cascade such as `pull` runs every stage it starts before the command returns,
and prints one line per stage. `app worker` is the boilerplate's: the same
queue with no web app, and the schedules fired from it.
`python -m app` is the same thing, for a checkout with no console script.
"""
from __future__ import annotations
import argparse
import json
import logging
import os
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app import config as config_mod
if TYPE_CHECKING:
from app.storage import Store
#: The one plugin this app ships lives outside the `app` package, in
#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a
#: user's own drop-in plugin would be. `companies` and `job pull` both want
#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as
#: well as the `Source` class, so the repo root goes on `sys.path` once, here,
#: rather than each caller guessing where `plugins/` is relative to `cwd`.
_ROOT = Path(__file__).resolve().parents[1]
if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
parser.add_argument("-c", "--config", default=None,
help="the YAML file to read (default: $APP_CONFIG, then config.yaml)")
parser.add_argument("-v", "--verbose", action="store_true")
sub = parser.add_subparsers(dest="command", required=True)
serve = sub.add_parser("serve", help="run the web app")
serve.add_argument("--host", default="", help="override config server.host")
serve.add_argument("--port", type=int, default=0, help="override config server.port")
serve.add_argument("--reload", action="store_true", help="restart on a source change")
job = sub.add_parser("job", help="queue a job, task or cascade and drain the queue")
job.add_argument("kind", help="pull, extract, embed, score, or any registered job")
job.add_argument("--payload", default="{}",
help="JSON spread over a job handler's keyword arguments")
worker = sub.add_parser("worker", help="drain the job table with no web app")
worker.add_argument("--workers", default="", help="resource=count,... e.g. cpu=2")
worker.add_argument("--until-idle", action="store_true",
help="run what is queued, then exit")
companies = sub.add_parser("companies", help="the target company list")
companies_sub = companies.add_subparsers(dest="action", required=True)
seed = companies_sub.add_parser(
"seed", help="import seed/companies.json, subscribing to each readable board")
seed.add_argument("--path", default="seed/companies.json")
add = companies_sub.add_parser("add", help="add one company from its board")
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
add.add_argument("name")
discover = companies_sub.add_parser(
"discover", help="find a company's board from its website; --add follows it")
discover.add_argument("name")
discover.add_argument("domain", help="its website, e.g. pleo.io")
discover.add_argument("--add", action="store_true",
help="add the company, with its board when one is readable")
companies_sub.add_parser("check", help="report boards that do not answer")
prune = companies_sub.add_parser(
"prune", help="unsubscribe every board with no open role in the region")
prune.add_argument("--in", dest="region", default="europe",
help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)")
prune.add_argument("--apply", action="store_true",
help="unsubscribe and close their roles; without it, only list them")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
search_add = searches_sub.add_parser(
"add", help="run a search once, then save it for every pull (or update it)")
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
search_add.add_argument("--q", required=True,
help="the query, in the provider's own syntax")
search_add.add_argument("--area", type=int, default=0,
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
search_add.add_argument("--country", default="",
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
search_add.add_argument("--pages", type=int, default=0,
help="result pages per pull (default 5, at most 10)")
searches_sub.add_parser("list", help="every saved search and its open roles")
search_remove = searches_sub.add_parser(
"remove", help="stop running a search and close its open roles")
search_remove.add_argument("name")
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
try:
if args.command == "serve":
return serve(args)
if args.command == "job":
return run_job(args)
if args.command == "worker":
return run_worker(args)
if args.command == "companies":
return run_companies(args)
if args.command == "searches":
return run_searches(args)
return init_db(args)
except (FileNotFoundError, config_mod.ConfigError) as exc:
print(exc, file=sys.stderr)
return 1
def serve(args: argparse.Namespace) -> int:
import uvicorn
config = config_mod.load(args.config) # a config error here, not in uvicorn
host = args.host or config.server.host
port = args.port or config.server.port
level = str(config.log_level).lower()
if args.reload:
# The reloader builds the app in a fresh process from an import
# string, and that process reads the config from $APP_CONFIG.
if args.config:
os.environ["APP_CONFIG"] = str(args.config)
uvicorn.run("app.main:app", host=host, port=port, reload=True, log_level=level)
return 0
from app.main import create_app
uvicorn.run(create_app(config), host=host, port=port, log_level=level)
return 0
def run_job(args: argparse.Namespace) -> int:
"""Queue `kind` (a cascade such as `pull`, a task from `pipeline.TASKS`,
or a job), run everything queued until the queue is idle, and print how
it ended: one line per task, from each task's final job. `pull` is a
cascade, so it prints a line for every stage it went through.
The queue fires no schedules: the command asked for one piece of work."""
from app import jobs
from app.storage import Store
from pipeline import FINAL, JOBS, TASKS, JobQueue, Jobs
cascade = jobs.cascade
try:
payload: dict[str, Any] = json.loads(args.payload or "{}")
except json.JSONDecodeError as exc:
print(f"--payload is not JSON: {exc}", file=sys.stderr)
return 2
chained = args.kind in cascade.CASCADES
if not chained and args.kind not in TASKS and args.kind not in JOBS:
known = sorted(set(JOBS) | set(TASKS))
print(f"no job kind {args.kind!r}. Have: {', '.join(known)}", file=sys.stderr)
return 2
config = config_mod.load(args.config)
store = Store(config.database)
queue = JobQueue(Jobs(str(config.database)), config, store,
workers=config.workers.model_dump(),
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
tz=config.timezone, schedules=False)
if chained or args.kind not in JOBS:
task_id = queue.start_task(cascade.chain(args.kind) if chained
else TASKS[args.kind]())
queue.drain()
started = queue.jobs.of_task(task_id)
if not started:
print(json.dumps({"task": args.kind, "result": "nothing to do"}))
return 0
whole = queue.jobs.of_cascade(started[0].cascade_id)
finals = [j for j in whole if j.kind == FINAL]
for final in finals:
# A task's own jobs say what the work did; the final job only
# tallies how they ended. Up to five, so a pull of twenty board
# batches stays one readable line.
work = [j.result for j in whole if j.task_id == final.task_id
and j.kind != FINAL][:5]
print(json.dumps({"id": final.id, "state": final.state,
"result": final.result, "work": work,
"error": final.error}))
return 0 if all(f.state == "done" for f in finals) else 1
queued = queue.enqueue(args.kind, payload)
queue.drain()
if queued is not None:
done = queue.jobs.get(queued.id)
if done is not None:
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
"error": done.error}))
return 0 if done.state == "done" else 1
return 0
def run_worker(args: argparse.Namespace) -> int:
"""`python -m pipeline.worker` with this app's settings filled in: the
bootstrap, the pools from `workers:`, and the timezone, without which a
`daily at 06:00` schedule would run at 06:00 on the machine's clock."""
from app.jobs import BOOTSTRAP
from pipeline.worker import main as worker_main
config = config_mod.load(args.config)
argv = ["--db", str(config.database), "--bootstrap", BOOTSTRAP,
"--bootstrap-arg", str(args.config or config.path or ""),
"--timezone", str(config.timezone)]
argv += ["--workers", args.workers or
",".join(f"{name}={n}" for name, n in config.workers.model_dump().items())]
if args.until_idle:
argv.append("--until-idle")
return worker_main(argv)
def run_companies(args: argparse.Namespace) -> int:
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "seed":
print(json.dumps(seed_companies(store, args.path)))
return 0
if args.action == "add":
return add_company(store, args.board, args.name)
if args.action == "discover":
return discover_company(store, args.name, args.domain, add=args.add)
if args.action == "prune":
return prune_companies(store, args.region, apply=args.apply)
return check_companies(store)
def seed_companies(store: Store, path: str) -> dict[str, int]:
"""Import `seed/companies.json`: one `companies` row per entry, and one
`ats` subscription for each entry with a readable `board`. An entry
without one is the point of the list as much as one with: a company worth
following whose careers site the app cannot read yet keeps its `ats`
host and `careers_url`, and the Companies section says so. A key that
already exists is skipped, so re-running only ever adds what is missing;
it never edits a row."""
from app.models import company_key
from plugins.ats import careers_url, is_known_target
data = json.loads(Path(path).read_text(encoding="utf-8"))
added = readable = skipped = 0
for row in data.get("companies", []):
name = row["name"]
key = row.get("key") or company_key(name)
if store.company(key) is not None:
skipped += 1
continue
board = row.get("board") or ""
if board and not is_known_target(board):
print(f"companies seed: {key}: {board!r} is not a known board; kept without it",
file=sys.stderr)
board = ""
host = board.partition(":")[0]
store.add_company(
name, key=key, domain=row.get("domain", ""),
careers_url=row.get("careers_url") or (careers_url(board) if board else ""),
ats=row.get("ats") or host, country=row.get("country", ""),
city=row.get("city", ""))
added += 1
if not board:
continue
try:
store.add_subscription("ats", board, label=name, company=name, board=host)
readable += 1
except ValueError as exc:
# The company row landed; the subscription is the only thing
# missing, and worth saying rather than losing silently.
print(f"companies seed: {key}: {exc}", file=sys.stderr)
return {"added": added, "readable": readable, "skipped": skipped,
"total": len(data.get("companies", []))}
def discover_company(store: Store, name: str, domain: str, *, add: bool) -> int:
"""`app companies discover "Pleo" pleo.io [--add]`: what the company's
site says its board is, as JSON. With `--add`, the company is added the
way the seed adds it: with its board when one is readable, as a row the
app cannot read yet otherwise."""
from app import jobs
found = jobs.discover.discover(name, domain)
out: dict[str, Any] = {"name": name, "domain": domain, "board": found.board,
"ats": found.ats, "careers_url": found.careers_url,
"evidence": found.evidence}
if add:
try:
out["key"], out["roles"] = jobs.sources.add_discovered(store, name, domain, found)
except ValueError as exc:
print(f"companies discover: {exc}", file=sys.stderr)
return 1
print(json.dumps(out))
return 0
def add_company(store: Store, board: str, name: str) -> int:
"""`app companies add greenhouse:acme "Acme"`: fetch the board first
and refuse if it does not answer, is not a known host, or the company
already exists. Port of seedgen's `-add`."""
from app import jobs
from app.models import company_key
try:
sub, found = jobs.sources.add_board(store, board, name)
except ValueError as exc:
print(f"companies add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": company_key(sub.company), "board": sub.target, "roles": found}))
return 0
def check_companies(store: Store) -> int:
"""`app companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer and exits 1 if any did not."""
from plugins.ats import fetch_board
subs = store.subscriptions(source="ats")
dead = []
for sub in subs:
try:
fetch_board(sub.target)
except Exception as exc:
dead.append((sub.target, sub.label, str(exc)))
for target, label, reason in dead:
print(f"{target} ({label}): {reason}")
print(f"checked {len(subs)} boards, {len(dead)} dead")
return 1 if dead else 0
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
"""`app companies prune --in DK,EU`: every fetched board none of whose open
roles is in the region, which is `europe` unless named. `--apply`
switches them off, which closes their open roles. `app companies seed`
never brings one back: it skips a company that exists. See
`app.jobs.sources.prune_plan` for which boards count."""
from app import jobs
try:
filled, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
print(f"companies prune: {exc}", file=sys.stderr)
return 1
for sub, count in plan:
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
result: dict[str, Any] = {"countries_filled": filled, "pruned": len(plan), "applied": apply}
if apply:
result |= jobs.sources.prune(store, region)
print(json.dumps(result))
return 0
def run_searches(args: argparse.Namespace) -> int:
from app import jobs
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "add":
return add_search(config, store, args)
if args.action == "remove":
sub = store.subscription_for("search", args.name)
if sub is None:
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
return 1
_, closed = jobs.sources.remove_search(store, sub.id)
print(json.dumps({"removed": sub.label, "roles_closed": closed}))
return 0
countries = store.open_countries_by_subscription()
for sub in store.subscriptions(source="search"):
removed = "" if sub.subscribed and sub.fetch else " (off)"
open_roles = sum(countries.get(sub.id, {}).values())
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
return 0
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
--area 1`: run the search once and save it only if it finds something.
A search of the same name is updated in place."""
from app import jobs
try:
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
country=args.country, where=args.where,
pages=args.pages)
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
params)
except ValueError as exc:
print(f"searches add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"search": sub.label, "target": sub.target, "roles": found,
"complete": complete}, ensure_ascii=False))
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from app.storage import LATEST, Store
config = config_mod.load(args.config)
store = Store(config.database)
applied = store.migrations_applied
print(f"{store.path}: schema v{LATEST}"
+ (f", applied {applied}" if applied else ""))
return 0
if __name__ == "__main__":
raise SystemExit(main())