768 lines
33 KiB
Python
768 lines
33 KiB
Python
"""The command line.
|
|
|
|
app serve [--host H] [--port P] [--reload]
|
|
app job pull|extract|embed|score [--payload JSON]
|
|
app worker [--workers http=8,cpu=1] [--until-idle]
|
|
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
|
|
app searches add|list|remove
|
|
app agents list|prompt
|
|
app stages [extract|summarise|enrich on|off]
|
|
app dupes
|
|
app prep [unmapped|unlinked] [--remap]
|
|
app db init
|
|
|
|
`app job` queues the work on a queue of its own and drains it here, so a
|
|
cascade such as `pull` runs every stage it starts before the command returns,
|
|
and prints one line per stage. `app worker` is the boilerplate's: the same
|
|
queue with no web app, and the schedules fired from it.
|
|
|
|
`python -m app` is the same thing, for a checkout with no console script.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from app import config as config_mod
|
|
|
|
if TYPE_CHECKING:
|
|
from app.models import Subscription
|
|
from app.storage import Store
|
|
|
|
#: The one plugin this app ships lives outside the `app` package, in
|
|
#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a
|
|
#: user's own drop-in plugin would be. `companies` and `job pull` both want
|
|
#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as
|
|
#: well as the `Source` class, so the repo root goes on `sys.path` once, here,
|
|
#: rather than each caller guessing where `plugins/` is relative to `cwd`.
|
|
_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_ROOT))
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
from app.models import HAND_REASONS
|
|
|
|
parser = argparse.ArgumentParser(
|
|
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
|
|
parser.add_argument("-c", "--config", default=None,
|
|
help="the YAML file to read (default: $APP_CONFIG, then config.yaml)")
|
|
parser.add_argument("-v", "--verbose", action="store_true")
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
|
|
serve = sub.add_parser("serve", help="run the web app")
|
|
serve.add_argument("--host", default="", help="override config server.host")
|
|
serve.add_argument("--port", type=int, default=0, help="override config server.port")
|
|
serve.add_argument("--reload", action="store_true", help="restart on a source change")
|
|
|
|
job = sub.add_parser("job", help="queue a job, task or cascade and drain the queue")
|
|
job.add_argument("kind", help="pull, extract, embed, score, or any registered job")
|
|
job.add_argument("--payload", default="{}",
|
|
help="JSON spread over a job handler's keyword arguments")
|
|
|
|
worker = sub.add_parser("worker", help="drain the job table with no web app")
|
|
worker.add_argument("--workers", default="", help="resource=count,... e.g. cpu=2")
|
|
worker.add_argument("--until-idle", action="store_true",
|
|
help="run what is queued, then exit")
|
|
|
|
companies = sub.add_parser("companies", help="the target company list")
|
|
companies_sub = companies.add_subparsers(dest="action", required=True)
|
|
seed = companies_sub.add_parser(
|
|
"seed", help="import seed/companies.json, and the boards it names for rows without one")
|
|
seed.add_argument("--path", default="seed/companies.json")
|
|
# One word after `add` is a company for the list checked by hand, two are
|
|
# a board and its company, which is what `add` meant before that list.
|
|
add = companies_sub.add_parser(
|
|
"add", help="add NAME to the list checked by hand, or with BOARD NAME follow a "
|
|
"company by its board, attaching or replacing an existing one's")
|
|
add.add_argument("first", metavar="NAME|BOARD",
|
|
help="the company, or host:slug, e.g. greenhouse:acme, before its name")
|
|
add.add_argument("second", metavar="NAME", nargs="?")
|
|
add.add_argument("--careers-url", default="", help="its careers page, http or https")
|
|
add.add_argument("--note", default="")
|
|
add.add_argument("--reason", default="added", choices=HAND_REASONS, metavar="REASON",
|
|
help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}")
|
|
edit = companies_sub.add_parser("edit", help="change a company's name, link, note or reason")
|
|
edit.add_argument("key")
|
|
edit.add_argument("--name")
|
|
edit.add_argument("--careers-url")
|
|
edit.add_argument("--note")
|
|
edit.add_argument("--reason", choices=HAND_REASONS, metavar="REASON",
|
|
help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}")
|
|
checked = companies_sub.add_parser(
|
|
"checked", help="record that its careers page was looked at just now")
|
|
checked.add_argument("key")
|
|
remove = companies_sub.add_parser(
|
|
"remove", help="delete a company with no board on and no open roles")
|
|
remove.add_argument("key")
|
|
companies_sub.add_parser(
|
|
"by-hand", help="the companies checked by hand, never checked first, then the "
|
|
"longest ago")
|
|
set_board = companies_sub.add_parser(
|
|
"set-board", help="attach a board to a company, or replace the one it has")
|
|
set_board.add_argument("key", help="the company key, e.g. trade-republic")
|
|
set_board.add_argument("board", help="host:slug, e.g. greenhouse:acme")
|
|
detach = companies_sub.add_parser(
|
|
"detach", help="stop reading a company's board and close its roles; the row stays")
|
|
detach.add_argument("key")
|
|
discover = companies_sub.add_parser(
|
|
"discover", help="find a company's board from its website; --add follows it")
|
|
discover.add_argument("name")
|
|
discover.add_argument("domain", help="its website, e.g. pleo.io")
|
|
discover.add_argument("--add", action="store_true",
|
|
help="add or update the company: its board when one is readable, "
|
|
"else the host it runs on")
|
|
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
|
|
size = companies_sub.add_parser(
|
|
"size", help="size a company's board again, or --missing every board without a size")
|
|
size.add_argument("key", nargs="?", default="", help="the company key, e.g. infineon")
|
|
size.add_argument("--missing", action="store_true", help="every board with no size yet")
|
|
|
|
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
|
searches_sub = searches.add_subparsers(dest="action", required=True)
|
|
search_add = searches_sub.add_parser(
|
|
"add", help="run a search once, then save it for every pull (or update it)")
|
|
search_add.add_argument("provider", choices=["jobindex", "adzuna", "platsbanken"])
|
|
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
|
|
search_add.add_argument("--q", required=True,
|
|
help="the query, in the provider's own syntax")
|
|
search_add.add_argument("--area", type=int, default=0,
|
|
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
|
|
search_add.add_argument("--country", default="",
|
|
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
|
|
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
|
|
search_add.add_argument("--region", default="",
|
|
help="platsbanken län id: CaRE_1nn_cSU is Skåne, empty all of Sweden")
|
|
search_add.add_argument("--pages", type=int, default=0,
|
|
help="result pages per pull (default 5, at most 10)")
|
|
searches_sub.add_parser("list", help="every saved search and its open roles")
|
|
search_remove = searches_sub.add_parser(
|
|
"remove", help="stop running a search and close its open roles")
|
|
search_remove.add_argument("name")
|
|
|
|
agents = sub.add_parser("agents", help="search agents and their prompts")
|
|
agents_sub = agents.add_subparsers(dest="action", required=True)
|
|
agents_sub.add_parser("list", help="every agent and its prompts")
|
|
prompt = agents_sub.add_parser(
|
|
"prompt", help="add a prompt to an agent, replace its text, or --remove it")
|
|
prompt.add_argument("agent", help="the agent's name")
|
|
prompt.add_argument("name", help='the prompt\'s name, e.g. "AI leadership"')
|
|
prompt.add_argument("text", nargs="?", default="",
|
|
help="what the agent looks for, in a sentence or a paragraph")
|
|
prompt.add_argument("--remove", action="store_true", help="remove the prompt instead")
|
|
|
|
stages = sub.add_parser(
|
|
"stages", help="show whether extract, summarise and enrich run, or switch one")
|
|
stages.add_argument("name", nargs="?", choices=["extract", "summarise", "enrich"])
|
|
stages.add_argument("state", nargs="?", choices=["on", "off"])
|
|
|
|
sub.add_parser("seniority", help="per host, how often the stated level equals the model's")
|
|
|
|
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
|
|
|
prep = sub.add_parser(
|
|
"prep", help="map the roles' host words and fold the copies, or list what is unmapped")
|
|
prep.add_argument("action", nargs="?", choices=["unmapped", "unlinked"],
|
|
help="list the host words no table maps, or the place texts no "
|
|
"place resolves, instead of running")
|
|
prep.add_argument("--remap", action="store_true",
|
|
help="map every role again, closed ones included")
|
|
|
|
db = sub.add_parser("db", help="database housekeeping")
|
|
db.add_argument("action", choices=["init"])
|
|
return parser
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
try:
|
|
if args.command == "serve":
|
|
return serve(args)
|
|
if args.command == "job":
|
|
return run_job(args)
|
|
if args.command == "worker":
|
|
return run_worker(args)
|
|
if args.command == "companies":
|
|
return run_companies(args)
|
|
if args.command == "searches":
|
|
return run_searches(args)
|
|
if args.command == "agents":
|
|
return run_agents(args)
|
|
if args.command == "stages":
|
|
return run_stages(args)
|
|
if args.command == "seniority":
|
|
return run_seniority(args)
|
|
if args.command == "dupes":
|
|
return run_dupes(args)
|
|
if args.command == "prep":
|
|
return run_prep(args)
|
|
return init_db(args)
|
|
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
|
print(exc, file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
def serve(args: argparse.Namespace) -> int:
|
|
import uvicorn
|
|
|
|
config = config_mod.load(args.config) # a config error here, not in uvicorn
|
|
host = args.host or config.server.host
|
|
port = args.port or config.server.port
|
|
level = str(config.log_level).lower()
|
|
if args.reload:
|
|
# The reloader builds the app in a fresh process from an import
|
|
# string, and that process reads the config from $APP_CONFIG.
|
|
if args.config:
|
|
os.environ["APP_CONFIG"] = str(args.config)
|
|
uvicorn.run("app.main:app", host=host, port=port, reload=True, log_level=level)
|
|
return 0
|
|
from app.main import create_app
|
|
|
|
uvicorn.run(create_app(config), host=host, port=port, log_level=level)
|
|
return 0
|
|
|
|
|
|
def run_job(args: argparse.Namespace) -> int:
|
|
"""Queue `kind` (a cascade such as `pull`, a task from `pipeline.TASKS`,
|
|
or a job), run everything queued until the queue is idle, and print how
|
|
it ended: one line per task, from each task's final job. `pull` is a
|
|
cascade, so it prints a line for every stage it went through.
|
|
|
|
The queue fires no schedules: the command asked for one piece of work."""
|
|
from app import jobs
|
|
from app.storage import Store
|
|
from pipeline import FINAL, JOBS, TASKS, JobQueue, Jobs
|
|
|
|
cascade = jobs.cascade
|
|
|
|
try:
|
|
payload: dict[str, Any] = json.loads(args.payload or "{}")
|
|
except json.JSONDecodeError as exc:
|
|
print(f"--payload is not JSON: {exc}", file=sys.stderr)
|
|
return 2
|
|
chained = args.kind in cascade.CASCADES
|
|
if not chained and args.kind not in TASKS and args.kind not in JOBS:
|
|
known = sorted(set(JOBS) | set(TASKS))
|
|
print(f"no job kind {args.kind!r}. Have: {', '.join(known)}", file=sys.stderr)
|
|
return 2
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
queue = JobQueue(Jobs(str(config.database)), config, store,
|
|
workers=config.workers.model_dump(),
|
|
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
|
|
tz=config.timezone, schedules=False)
|
|
if chained or args.kind not in JOBS:
|
|
first = cascade.chain(args.kind) if chained else TASKS[args.kind]()
|
|
if args.kind == "pull":
|
|
# The same claim the `pull` job makes, so a pull here and one in
|
|
# the server's worker cannot both fetch.
|
|
run = jobs.pull.claim_run(store)
|
|
if not run:
|
|
print(json.dumps({"task": "pull", "result": jobs.pull.ALREADY_RUNNING,
|
|
"running": jobs.pull.running(store)}))
|
|
return 1
|
|
first = cascade.pull_chain(run)
|
|
task_id = queue.start_task(first)
|
|
queue.drain()
|
|
started = queue.jobs.of_task(task_id)
|
|
if not started:
|
|
print(json.dumps({"task": args.kind, "result": "nothing to do"}))
|
|
return 0
|
|
whole = queue.jobs.of_cascade(started[0].cascade_id)
|
|
finals = [j for j in whole if j.kind == FINAL]
|
|
for final in finals:
|
|
# A task's own jobs say what the work did; the final job only
|
|
# tallies how they ended. Up to five, so a pull of twenty board
|
|
# batches stays one readable line.
|
|
work = [j.result for j in whole if j.task_id == final.task_id
|
|
and j.kind != FINAL][:5]
|
|
print(json.dumps({"id": final.id, "state": final.state,
|
|
"result": final.result, "work": work,
|
|
"error": final.error}))
|
|
return 0 if all(f.state == "done" for f in finals) else 1
|
|
queued = queue.enqueue(args.kind, payload)
|
|
queue.drain()
|
|
if queued is not None:
|
|
done = queue.jobs.get(queued.id)
|
|
if done is not None:
|
|
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
|
|
"error": done.error}))
|
|
return 0 if done.state == "done" else 1
|
|
return 0
|
|
|
|
|
|
def run_worker(args: argparse.Namespace) -> int:
|
|
"""`python -m pipeline.worker` with this app's settings filled in: the
|
|
bootstrap, the pools from `workers:`, and the timezone, without which a
|
|
`daily at 06:00` schedule would run at 06:00 on the machine's clock."""
|
|
from app.jobs import BOOTSTRAP
|
|
from pipeline.worker import main as worker_main
|
|
|
|
config = config_mod.load(args.config)
|
|
argv = ["--db", str(config.database), "--bootstrap", BOOTSTRAP,
|
|
"--bootstrap-arg", str(args.config or config.path or ""),
|
|
"--timezone", str(config.timezone)]
|
|
argv += ["--workers", args.workers or
|
|
",".join(f"{name}={n}" for name, n in config.workers.model_dump().items())]
|
|
if args.until_idle:
|
|
argv.append("--until-idle")
|
|
return worker_main(argv)
|
|
|
|
|
|
def run_companies(args: argparse.Namespace) -> int:
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "seed":
|
|
return seed_companies(store, args.path)
|
|
if args.action == "add":
|
|
return add_company(store, args)
|
|
if args.action == "edit":
|
|
return edit_company(store, args)
|
|
if args.action == "checked":
|
|
return company_checked(store, args.key)
|
|
if args.action == "remove":
|
|
return remove_company(store, args.key)
|
|
if args.action == "by-hand":
|
|
return list_by_hand(store)
|
|
if args.action == "set-board":
|
|
return set_board(store, args.key, args.board)
|
|
if args.action == "detach":
|
|
return detach_board(store, args.key)
|
|
if args.action == "discover":
|
|
return discover_company(store, args.name, args.domain, add=args.add)
|
|
if args.action == "size":
|
|
return size_boards(store, args.key, missing=args.missing)
|
|
return check_companies(store)
|
|
|
|
|
|
def seed_companies(store: Store, path: str) -> int:
|
|
"""`app companies seed`: `app.jobs.sources.seed_companies`, its counts as
|
|
JSON and each entry that did not land whole on stderr."""
|
|
from app import jobs
|
|
|
|
result = jobs.sources.seed_companies(store, path)
|
|
for problem in result.pop("problems"):
|
|
print(f"companies seed: {problem}", file=sys.stderr)
|
|
print(json.dumps(result))
|
|
return 0
|
|
|
|
|
|
def discover_company(store: Store, name: str, domain: str, *, add: bool) -> int:
|
|
"""`app companies discover "Pleo" pleo.io [--add]`: what the company's
|
|
site says its board is, as JSON. With `--add`, the company is added or
|
|
updated: its board attached or replaced when one is readable, else the
|
|
host and slug it runs on recorded; see `add_discovered`."""
|
|
from app import jobs
|
|
|
|
found = jobs.discover.discover(name, domain)
|
|
out: dict[str, Any] = {"name": name, "domain": domain, "board": found.board,
|
|
"ats": found.ats, "ats_slug": found.ats_slug,
|
|
"careers_url": found.careers_url,
|
|
"evidence": found.evidence}
|
|
if add:
|
|
try:
|
|
out["key"], out["roles"] = jobs.sources.add_discovered(store, name, domain, found)
|
|
except ValueError as exc:
|
|
print(f"companies discover: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps(out))
|
|
return 0
|
|
|
|
|
|
def add_company(store: Store, args: argparse.Namespace) -> int:
|
|
"""`app companies add "Acme" [--careers-url URL] [--note TEXT] [--reason
|
|
REASON]` puts a company with no board on the list checked by hand;
|
|
`app companies add greenhouse:acme "Acme"` follows it by its board."""
|
|
if args.second is not None:
|
|
return add_company_by_board(store, args.first, args.second)
|
|
from app import jobs
|
|
|
|
try:
|
|
company = jobs.sources.add_by_hand(store, args.first, careers_url=args.careers_url,
|
|
notes=args.note, hand_reason=args.reason)
|
|
except ValueError as exc:
|
|
print(f"companies add: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": company.key, "hand_reason": company.hand_reason}))
|
|
return 0
|
|
|
|
|
|
def add_company_by_board(store: Store, board: str, name: str) -> int:
|
|
"""`app companies add greenhouse:acme "Acme"`: fetch the board first
|
|
and refuse if it does not answer or is not a known host. A company that
|
|
exists gets the board attached or replaced. Port of seedgen's `-add`."""
|
|
from app import jobs
|
|
from app.models import company_key
|
|
|
|
try:
|
|
sub, found = jobs.sources.add_board(store, board, name)
|
|
except ValueError as exc:
|
|
print(f"companies add: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": company_key(sub.company), "board": sub.target, "roles": found}))
|
|
return 0
|
|
|
|
|
|
def edit_company(store: Store, args: argparse.Namespace) -> int:
|
|
"""`app companies edit acme --careers-url https://acme.example/jobs`: only
|
|
the options given change; see `jobs.sources.edit_company`."""
|
|
from app import jobs
|
|
|
|
given = {"name": args.name, "careers_url": args.careers_url, "notes": args.note,
|
|
"hand_reason": args.reason}
|
|
try:
|
|
company = jobs.sources.edit_company(
|
|
store, args.key, **{f: v for f, v in given.items() if v is not None})
|
|
except KeyError:
|
|
print(f"companies edit: no company {args.key!r}", file=sys.stderr)
|
|
return 1
|
|
except ValueError as exc:
|
|
print(f"companies edit: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": company.key, "name": company.name,
|
|
"careers_url": company.careers_url, "notes": company.notes,
|
|
"hand_reason": company.hand_reason}))
|
|
return 0
|
|
|
|
|
|
def company_checked(store: Store, key: str) -> int:
|
|
"""`app companies checked acme`: its careers page was looked at just now,
|
|
which the Watchlist keeps as `careers_opened_at`, the same stamp opening
|
|
the page from there sets."""
|
|
if store.company(key) is None:
|
|
print(f"companies checked: no company {key!r}", file=sys.stderr)
|
|
return 1
|
|
company = store.update_company(key, careers_opened_at=store.now())
|
|
print(json.dumps({"key": key, "careers_opened_at": company.careers_opened_at}))
|
|
return 0
|
|
|
|
|
|
def remove_company(store: Store, key: str) -> int:
|
|
"""`app companies remove acme`: refused for a company still read, see
|
|
`jobs.sources.remove_company`."""
|
|
from app import jobs
|
|
|
|
try:
|
|
jobs.sources.remove_company(store, key)
|
|
except KeyError:
|
|
print(f"companies remove: no company {key!r}", file=sys.stderr)
|
|
return 1
|
|
except ValueError as exc:
|
|
print(f"companies remove: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": key, "removed": True}))
|
|
return 0
|
|
|
|
|
|
def list_by_hand(store: Store) -> int:
|
|
"""`app companies by-hand`: the companies with no board shown that are not
|
|
known from a search only, one line each, never opened first and then the
|
|
longest ago; `watch` is the Watchlist's rating, yes, no or a dash for
|
|
unrated. See `jobs.sources.by_hand`."""
|
|
from app import jobs
|
|
|
|
rows = jobs.sources.by_hand(store)
|
|
for c in rows:
|
|
watch = "-" if c.watch is None else "yes" if c.watch else "no"
|
|
opened = c.careers_opened_at[:10] or "never"
|
|
print(f"{c.name:30} {c.hand_reason or '-':8} {watch:4} {opened:10} {c.careers_url}")
|
|
print(f"{len(rows)} by hand")
|
|
return 0
|
|
|
|
|
|
def set_board(store: Store, key: str, board: str) -> int:
|
|
"""`app companies set-board trade-republic greenhouse:traderepublic`:
|
|
attach the board to an existing company or replace the one it has."""
|
|
from app import jobs
|
|
|
|
try:
|
|
sub, found, closed = jobs.sources.set_board(store, key, board)
|
|
except ValueError as exc:
|
|
print(f"companies set-board: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": key, "board": sub.target, "roles": found, "closed": closed}))
|
|
return 0
|
|
|
|
|
|
def detach_board(store: Store, key: str) -> int:
|
|
"""`app companies detach acme`: stop reading its board, close its roles."""
|
|
from app import jobs
|
|
|
|
try:
|
|
closed = jobs.sources.detach_board(store, key)
|
|
except KeyError:
|
|
print(f"companies detach: no company {key!r}", file=sys.stderr)
|
|
return 1
|
|
except ValueError as exc:
|
|
print(f"companies detach: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"key": key, "closed": closed}))
|
|
return 0
|
|
|
|
|
|
def size_boards(store: Store, key: str, *, missing: bool) -> int:
|
|
"""`app companies size infineon`: size its board again and keep it, as
|
|
the Companies row's Recheck does, and print the size and what a pull asks
|
|
the board for. `--missing` sizes every board shown that has no size yet,
|
|
a line each, and exits 1 if any did not answer. The next pull reads a
|
|
board so; nothing is queued from here."""
|
|
from app import jobs
|
|
|
|
if bool(key) == missing:
|
|
print("companies size: name a company key, or --missing", file=sys.stderr)
|
|
return 2
|
|
if missing:
|
|
subs = [s for s in store.subscriptions(source="ats") if s.show]
|
|
known = jobs.reach.sizes(store, subs)
|
|
subs = [s for s in subs if s.target not in known]
|
|
else:
|
|
sub = store.company_subscription(key)
|
|
if sub is None or not sub.show:
|
|
print(f"companies size: {key!r} has no board", file=sys.stderr)
|
|
return 1
|
|
subs = [sub]
|
|
sized: list[tuple[Subscription, int]] = []
|
|
for sub in subs:
|
|
try:
|
|
sized.append((sub, jobs.sources.size_board(store, sub.target)))
|
|
except ValueError as exc:
|
|
print(f"companies size: {exc}", file=sys.stderr)
|
|
plan = jobs.reach.plan(store, [sub for sub, _size in sized])
|
|
for sub, size in sized:
|
|
print(json.dumps({"board": sub.target, "size": size,
|
|
"reach": jobs.reach.line(plan, sub)}))
|
|
dead = len(subs) - len(sized)
|
|
if missing:
|
|
print(f"sized {len(sized)} of {len(subs)} boards without a size")
|
|
return 1 if dead else 0
|
|
|
|
|
|
def check_companies(store: Store) -> int:
|
|
"""`app companies check`: every `ats` board, fetched once. Prints a
|
|
line per board that did not answer now, answered incompletely, or has
|
|
been failing in the pull, with what the pull recorded beside the live
|
|
result, and exits 1 if any did not answer now."""
|
|
from app import jobs
|
|
from plugins.ats import failure_reason, fetch_board
|
|
|
|
subs = [s for s in store.subscriptions(source="ats") if s.show]
|
|
failing = jobs.pull.failing_boards(store)
|
|
dead = 0
|
|
for sub in subs:
|
|
try:
|
|
rows, complete = fetch_board(sub.target)
|
|
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
|
|
except Exception as exc:
|
|
complete, live = False, f"not answering: {failure_reason(exc)}"
|
|
dead += 1
|
|
since, reason = failing.get(sub.target, ("", ""))
|
|
if complete and not since:
|
|
continue
|
|
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
|
|
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
|
|
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
|
|
return 1 if dead else 0
|
|
|
|
|
|
def run_searches(args: argparse.Namespace) -> int:
|
|
from app import jobs
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "add":
|
|
return add_search(config, store, args)
|
|
if args.action == "remove":
|
|
sub = store.subscription_for("search", args.name)
|
|
if sub is None:
|
|
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
|
|
return 1
|
|
_, closed = jobs.sources.remove_search(store, sub.id)
|
|
print(json.dumps({"removed": sub.label, "roles_closed": closed}))
|
|
return 0
|
|
countries = store.open_countries_by_subscription()
|
|
for sub in store.subscriptions(source="search"):
|
|
removed = "" if sub.subscribed and sub.fetch else " (off)"
|
|
open_roles = sum(countries.get(sub.id, {}).values())
|
|
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
|
|
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
|
|
return 0
|
|
|
|
|
|
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
|
|
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
|
|
--area 1`: run the search once and save it only if it finds something.
|
|
A search of the same name is updated in place."""
|
|
from app import jobs
|
|
|
|
try:
|
|
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
|
|
country=args.country, where=args.where,
|
|
region=args.region, pages=args.pages)
|
|
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
|
|
params)
|
|
except ValueError as exc:
|
|
print(f"searches add: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps({"search": sub.label, "target": sub.target, "roles": found,
|
|
"complete": complete}, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
def run_agents(args: argparse.Namespace) -> int:
|
|
"""`app agents list` prints every agent and its prompts. `app agents
|
|
prompt "Wolf, next role" "AI leadership" "Head of AI ..."` adds that
|
|
prompt at the end, or replaces the text of the one of that name in its
|
|
place; `--remove` removes it. The agent's other fields stay as they are.
|
|
Scores follow at the next `score`."""
|
|
from app.models import Prompt
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "list":
|
|
for each in store.agents():
|
|
print(each.name + ("" if each.enabled else " (off)"))
|
|
for p in each.prompts:
|
|
text = " ".join(p.text.split())
|
|
print(f" {p.name}: {text[:100] + '…' if len(text) > 100 else text}")
|
|
return 0
|
|
agent = next((a for a in store.agents() if a.name == args.agent), None)
|
|
if agent is None:
|
|
print(f"agents prompt: no agent named {args.agent!r}", file=sys.stderr)
|
|
return 1
|
|
named = args.name.strip().casefold()
|
|
prompts = [p for p in agent.prompts if not (args.remove and p.name.casefold() == named)]
|
|
if args.remove and len(prompts) == len(agent.prompts):
|
|
print(f"agents prompt: {agent.name!r} has no prompt named {args.name!r}",
|
|
file=sys.stderr)
|
|
return 1
|
|
if not args.remove:
|
|
try:
|
|
new = Prompt(name=args.name, text=args.text)
|
|
except ValueError as exc:
|
|
print(f"agents prompt: {exc}", file=sys.stderr)
|
|
return 1
|
|
at = next((n for n, p in enumerate(prompts) if p.name.casefold() == named), None)
|
|
if at is None:
|
|
prompts.append(new)
|
|
else:
|
|
prompts[at] = new
|
|
saved = store.save_agent(agent.name, gates=agent.gates, weights=agent.weights,
|
|
enabled=agent.enabled, agent_id=agent.id, limits=agent.limits,
|
|
prompts=prompts)
|
|
print(json.dumps({"agent": saved.name, "prompts": [p.name for p in saved.prompts]},
|
|
ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
def run_stages(args: argparse.Namespace) -> int:
|
|
"""`app stages` prints what `GET /api/stages` returns; `app stages extract
|
|
on` stores the switch through the function `PUT /api/stages/extract`
|
|
calls, and prints the result."""
|
|
from app import jobs
|
|
from app.storage import Store
|
|
|
|
if bool(args.name) != bool(args.state):
|
|
print("stages: give a stage and on or off, or neither", file=sys.stderr)
|
|
return 1
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.name:
|
|
jobs.stages.set_stage(store, args.name, args.state == "on")
|
|
found = [jobs.stages.describe_one(config, store, args.name)]
|
|
else:
|
|
found = jobs.stages.describe(config, store)
|
|
for stage in found:
|
|
where = "set by the switch" if stage.source == "stored" else "config.yaml's default"
|
|
print(f"{stage.name:10} {'on ' if stage.on else 'off'} {where}; "
|
|
f"{stage.model}; {stage.answer}")
|
|
return 0
|
|
|
|
|
|
def run_seniority(args: argparse.Namespace) -> int:
|
|
"""`app seniority`: per host, the roles that have both a stated and a
|
|
model level and how many of them agree, `Store.seniority_agreement`, as
|
|
a table. For deciding which host's stated level to trust."""
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
found = Store(config.database).seniority_agreement()
|
|
if not found:
|
|
print("no role has a stated level yet")
|
|
return 0
|
|
print(f"{'host':20} {'compared':>8} {'agreed':>7} {'rate':>6} {'unread':>7}")
|
|
for h in found:
|
|
rate = "-" if h["rate"] is None else f"{h['rate']:.0%}"
|
|
print(f"{h['host'] or '(none)':20} {h['compared']:8} {h['agreed']:7} {rate:>6} "
|
|
f"{h['stated_only']:7}")
|
|
return 0
|
|
|
|
|
|
def run_dupes(args: argparse.Namespace) -> int:
|
|
"""`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`,
|
|
now. Prints the groups (kept roles with a copy), the copies the feed
|
|
hides and the rows whose `dup_of` changed, as JSON."""
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
counts = Store(config.database).rebuild_dupes()
|
|
print(json.dumps(counts._asdict()))
|
|
return 0
|
|
|
|
|
|
def run_prep(args: argparse.Namespace) -> int:
|
|
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
|
|
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
|
|
host words no table maps, per field and host, most roles first; `app
|
|
prep unlinked` the place texts of open roles no place resolves, with the
|
|
countries and companies of their roles."""
|
|
from app import prep
|
|
from app.storage import Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
if args.action == "unmapped":
|
|
found = store.unmapped()
|
|
if not found:
|
|
print("every host word is mapped")
|
|
for w in found:
|
|
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
|
|
return 0
|
|
if args.action == "unlinked":
|
|
linked = store.unlinked_places()
|
|
if not linked:
|
|
print("every place text is linked")
|
|
for p in linked:
|
|
where = " ".join(f"{c or '-'}:{n}" for c, n in p["countries"][:3])
|
|
print(f"{p['roles']:6} {p['text'][:40]:40} {where:20} {', '.join(p['companies'])}")
|
|
return 0
|
|
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
|
|
return 0
|
|
|
|
|
|
def init_db(args: argparse.Namespace) -> int:
|
|
"""Create the file, or bring it to the current schema."""
|
|
from app.storage import LATEST, Store
|
|
|
|
config = config_mod.load(args.config)
|
|
store = Store(config.database)
|
|
applied = store.migrations_applied
|
|
print(f"{store.path}: schema v{LATEST}"
|
|
+ (f", applied {applied}" if applied else ""))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|